mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-08-05 09:34:36 +00:00
Compare commits
114
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7e5fba910 | ||
|
|
93df935446 | ||
|
|
27f53948ca | ||
|
|
48c69857ed | ||
|
|
c971862ac6 | ||
|
|
22b35ff93d | ||
|
|
2f2820eb33 | ||
|
|
a6c129c02d | ||
|
|
a2e2a5c662 | ||
|
|
8e8a9a46b6 | ||
|
|
47d0d87a88 | ||
|
|
3b2c5a0671 | ||
|
|
0609c5ed77 | ||
|
|
694bc04623 | ||
|
|
da859f2f84 | ||
|
|
dd8c4a9a8a | ||
|
|
2b4e4c2430 | ||
|
|
8c3d8f52ba | ||
|
|
e903aa3a9f | ||
|
|
f3eefc2638 | ||
|
|
3463149b38 | ||
|
|
dc13cde00c | ||
|
|
55dba5136c | ||
|
|
c0aad3b3ab | ||
|
|
19236ecc9d | ||
|
|
7a2b485af6 | ||
|
|
2ec6b27200 | ||
|
|
f9980e5c66 | ||
|
|
98fe5a37cd | ||
|
|
60566313dc | ||
|
|
443c910b9e | ||
|
|
88d301398b | ||
|
|
4cc9aef61f | ||
|
|
3befe063f4 | ||
|
|
f8adaee362 | ||
|
|
895ea90bfa | ||
|
|
c4d27ba02a | ||
|
|
5ea5024608 | ||
|
|
bb2a05b67c | ||
|
|
c75dee147a | ||
|
|
e2f89a2645 | ||
|
|
48c11eb7bc | ||
|
|
c829685f80 | ||
|
|
16fc548b48 | ||
|
|
c5da73f1e9 | ||
|
|
06dba681c5 | ||
|
|
f96acbb99c | ||
|
|
0a00527626 | ||
|
|
73a3f12469 | ||
|
|
39f801561b | ||
|
|
ff55788e2d | ||
|
|
e38cd32fa3 | ||
|
|
423e2cba73 | ||
|
|
1138cdd4ef | ||
|
|
f15eab9f10 | ||
|
|
344c3c5f21 | ||
|
|
4c6bc49e07 | ||
|
|
1d4b8d7cb2 | ||
|
|
1131c2d583 | ||
|
|
7108d554c8 | ||
|
|
1eddf89d52 | ||
|
|
b0c70ecddc | ||
|
|
a2cedd6769 | ||
|
|
7b0ade3987 | ||
|
|
81c0ff57e9 | ||
|
|
2410a4a6ce | ||
|
|
1b09639265 | ||
|
|
7394b10e75 | ||
|
|
4c292580e8 | ||
|
|
04879cad56 | ||
|
|
ab80657507 | ||
|
|
b94d95fc2b | ||
|
|
27f81dbf42 | ||
|
|
66ba31d0df | ||
|
|
16679c1f4e | ||
|
|
2ed20b1b85 | ||
|
|
92246b78d0 | ||
|
|
6023c03959 | ||
|
|
dbe7a53afd | ||
|
|
e0c74e3a46 | ||
|
|
97dc10a8ad | ||
|
|
a9a6381614 | ||
|
|
040799a4d7 | ||
|
|
87850c97b9 | ||
|
|
586af15a1b | ||
|
|
3e906605a0 | ||
|
|
4defe4f227 | ||
|
|
f8125a8a2d | ||
|
|
999a5634fa | ||
|
|
2c218cce49 | ||
|
|
fa0b366f9a | ||
|
|
1957e716a3 | ||
|
|
69f19ff991 | ||
|
|
8b942f3c14 | ||
|
|
09e1c7bf2d | ||
|
|
cc2a96e2ef | ||
|
|
93ab1d927b | ||
|
|
39970d8bee | ||
|
|
d7c401d204 | ||
|
|
d44b98fd0d | ||
|
|
6fa3610423 | ||
|
|
65702171e4 | ||
|
|
65abcbce92 | ||
|
|
40153d4c36 | ||
|
|
40bf976fbc | ||
|
|
eae20f04a7 | ||
|
|
acb630f6cf | ||
|
|
1230d528de | ||
|
|
d23c90b939 | ||
|
|
d8db2a3051 | ||
|
|
0bbbf902cd | ||
|
|
7ed18a9d02 | ||
|
|
dc25659cff | ||
|
|
349d8dd009 |
@@ -22,6 +22,19 @@ ROUTSTR_SECRET_KEY=
|
||||
|
||||
# Database
|
||||
# DATABASE_URL=sqlite+aiosqlite:///keys.db
|
||||
# Pool controls are validated at boot, sourced only from the environment, and
|
||||
# logged at startup. Keep total capacity across all workers below the database
|
||||
# connection limit. Pre-ping is automatic for networked backends; SQLite may
|
||||
# explicitly opt in if desired.
|
||||
# DATABASE_POOL_SIZE=10
|
||||
# DATABASE_MAX_OVERFLOW=20
|
||||
# DATABASE_POOL_TIMEOUT=15
|
||||
# DATABASE_POOL_RECYCLE=1800
|
||||
# DATABASE_POOL_PRE_PING=false
|
||||
# Warn when a checkout is held this many seconds.
|
||||
# DATABASE_POOL_HOLD_WARN_SECONDS=10
|
||||
# SQLite serialises writes; increasing its pool can trade pool timeouts for
|
||||
# "database is locked" errors rather than increasing write throughput.
|
||||
|
||||
# Node Information
|
||||
# NAME=My Routstr Node
|
||||
@@ -31,7 +44,12 @@ ROUTSTR_SECRET_KEY=
|
||||
# RELAYS="wss://relay.damus.io,wss://relay.nostr.band,wss://eden.nostr.land,wss://relay.routstr.com"
|
||||
# ENABLE_ANALYTICS_SHARING=true
|
||||
# CASHU_MINTS="https://mint.minibits.cash/Bitcoin,https://mint.cubabitcoin.org,https://ecashmint.otrta.me"
|
||||
# MINT_OPERATION_CONCURRENCY=4
|
||||
# MINT_OPERATION_TIMEOUT_SECONDS=30
|
||||
# MINT_MAX_CONCURRENCY=4
|
||||
# MINT_RETRY_MAX_ATTEMPTS=3
|
||||
# RECEIVE_LN_ADDRESS=
|
||||
# REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS=900
|
||||
|
||||
# Custom Pricing Configuration
|
||||
# MODEL_BASED_PRICING=true
|
||||
|
||||
@@ -327,6 +327,62 @@ GET /v1/models
|
||||
}
|
||||
```
|
||||
|
||||
### List Model Paths
|
||||
|
||||
Get the selectable upstream routes for each advertised model. This endpoint is
|
||||
discovery-only; request-side selection will be added separately.
|
||||
|
||||
```http
|
||||
GET /v1/models/paths
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "anthropic/claude-sonnet-4",
|
||||
"paths": [
|
||||
{
|
||||
"path": "url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=anthropic%2Fclaude-sonnet-4",
|
||||
"provider": {"id": 12, "slug": "anthropic-primary", "type": "anthropic"},
|
||||
"endpoint": null
|
||||
},
|
||||
{
|
||||
"path": "url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=anthropic%2Fclaude-sonnet-4&endpoint=google-vertex%2Fus",
|
||||
"provider": {"id": 42, "slug": "openrouter-main", "type": "openrouter"},
|
||||
"endpoint": {"tag": "google-vertex/us", "name": "Google"}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"updated_at": 1753500000
|
||||
}
|
||||
```
|
||||
|
||||
`path` is an opaque, percent-encoded selector. Clients must store and return it
|
||||
unchanged rather than parsing or reconstructing it. It identifies the exact
|
||||
configured route with `url`, `provider-id`, and `model-id`. To avoid exposing
|
||||
private network details, a configured private IP address or any URL with an
|
||||
explicit port is advertised as `http://localhost`. OpenRouter routes additionally
|
||||
preserve the exact machine-readable endpoint `tag`. Provider slugs/types and
|
||||
endpoint names remain display data. When request-side selection is implemented,
|
||||
an endpoint tag must not silently fall back to another backend.
|
||||
|
||||
### List Paths for One Model
|
||||
|
||||
Use the exact model ID advertised by `/v1/models`. The query parameter safely
|
||||
supports IDs containing `/`.
|
||||
|
||||
```http
|
||||
GET /v1/models/paths/model?model_id=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
The response uses the same path objects and `updated_at` field as the collection
|
||||
endpoint. An unknown model returns `404 Model not found`. A known model whose
|
||||
paths have not been discovered yet returns `200` with an empty `data` array.
|
||||
|
||||
## Wallet Management
|
||||
|
||||
### Create Wallet (Coming Soon)
|
||||
|
||||
@@ -100,6 +100,7 @@ All errors follow a consistent format:
|
||||
Standard OpenAI-compatible endpoints:
|
||||
|
||||
- **Models**: `/v1/models`
|
||||
- **Model paths**: `/v1/models/paths`, `/v1/models/paths/model?model_id=...`
|
||||
- **Responses**: `/v1/responses`
|
||||
- **Chat Completions**: `/v1/chat/completions`
|
||||
- **Embeddings**: `/v1/embeddings`
|
||||
@@ -302,7 +303,7 @@ Get node metadata:
|
||||
GET /v1/info
|
||||
```
|
||||
|
||||
Supported models and pricing are available at `/v1/models`.
|
||||
Supported models and pricing are available at `/v1/models`. Upstream provider path discovery is available at `/v1/models/paths` and `/v1/models/paths/model?model_id=...`.
|
||||
|
||||
## Next Steps
|
||||
|
||||
|
||||
@@ -48,6 +48,68 @@ Connect to your AI provider(s):
|
||||
| **Upstream URL** | API endpoint (e.g., `https://api.openai.com/v1`) |
|
||||
| **API Key** | Your provider's API key |
|
||||
|
||||
### PPQ Auto Top-up
|
||||
|
||||
PPQ providers can automatically purchase more credits when their USD balance
|
||||
falls below a configured threshold. Configure this per provider in the Admin
|
||||
Dashboard by editing a **PPQ.AI** provider and opening **PPQ Auto Top-up**.
|
||||
There are no environment variables for this feature.
|
||||
|
||||
#### Requirements
|
||||
|
||||
Before enabling auto top-up, make sure that:
|
||||
|
||||
- the PPQ provider has a valid API key;
|
||||
- at least one trusted Cashu mint is configured;
|
||||
- the node wallet has enough **node-owned** funds at one mint to pay the
|
||||
Lightning invoice; client balances are never used; and
|
||||
- the node has a current BTC/USD price for validating the invoice amount.
|
||||
|
||||
| Setting | Description |
|
||||
| ------- | ----------- |
|
||||
| **Enable Auto Top-up** | Enables automatic PPQ credit purchases for this provider. |
|
||||
| **When credits are below (USD)** | Starts a top-up when the reported PPQ balance is below this positive USD value. |
|
||||
| **Purchase this amount (USD)** | Amount of PPQ credit to buy per top-up. Must be a whole number from **1 to 500 USD**. |
|
||||
|
||||
For example, a threshold of `5` and purchase amount of `20` buys 20 USD of
|
||||
credit when the PPQ balance drops below 5 USD.
|
||||
|
||||
#### How it works
|
||||
|
||||
The worker checks eligible providers approximately once per minute. When the
|
||||
balance is below the threshold, it:
|
||||
|
||||
1. verifies the node has enough owner funds before creating an invoice;
|
||||
2. requests a USD-denominated Lightning top-up invoice from PPQ;
|
||||
3. rejects expired, mismatched, or unexpectedly expensive invoices (more than
|
||||
10% above the local BTC/USD estimate);
|
||||
4. pays from the configured Cashu mint with sufficient owner funds; and
|
||||
5. waits for PPQ to confirm that the credit settled.
|
||||
|
||||
Only one attempt can be active for a provider. An attempt that was active at
|
||||
the start of a cycle suppresses another top-up for that entire cycle, even if
|
||||
PPQ reports it settled immediately. This prevents a temporarily stale PPQ
|
||||
balance from causing a duplicate purchase.
|
||||
|
||||
Completed PPQ payments appear in the dashboard transaction history with source
|
||||
`ppq_auto_topup`. The payment record is separate from the internal claim used
|
||||
to prevent concurrent attempts.
|
||||
|
||||
#### Payment recovery
|
||||
|
||||
If the Cashu mint paid the invoice but PPQ settlement cannot be confirmed, the
|
||||
provider card shows **Auto top-up needs review**. A payment still owned by a
|
||||
running worker is shown as **Paying invoice** and cannot be released.
|
||||
|
||||
Before choosing **Release top-up**, manually verify both PPQ and the Cashu mint.
|
||||
Release the claim only when the previous Lightning payment is definitively
|
||||
unable to settle. Releasing an ambiguous payment allows the next cycle to try
|
||||
again and can therefore cause a duplicate top-up.
|
||||
|
||||
Disabling auto top-up prevents new purchases, but the node continues to
|
||||
reconcile an already active payment until it reaches a safe terminal state or
|
||||
requires operator review.
|
||||
|
||||
### Node Identity
|
||||
|
||||
How your node appears to clients:
|
||||
@@ -136,12 +198,23 @@ Use environment variables for:
|
||||
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
|
||||
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
|
||||
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
|
||||
| `MINT_OPERATION_CONCURRENCY` | Concurrent mint/unit balance reads | `4` |
|
||||
| `MINT_OPERATION_TIMEOUT_SECONDS` | Per-attempt timeout for mint network calls | `30` |
|
||||
| `MINT_MAX_CONCURRENCY` | Concurrent operations allowed per mint (`0` disables the limit) | `4` |
|
||||
| `MINT_RETRY_MAX_ATTEMPTS` | Retries after a timeout or HTTP 429 (`0` disables retries) | `3` |
|
||||
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
|
||||
| `MIN_PAYOUT_SAT` | Min payout balance in sats (applies to all mints) | `210` |
|
||||
| `PAYOUT_INTERVAL_SECONDS` | Payout loop interval (seconds) | `900` |
|
||||
| `TOR_PROXY_URL` | SOCKS5 proxy for Tor | `socks5://127.0.0.1:9050` |
|
||||
| `CORS_ORIGINS` | Allowed CORS origins | `*` |
|
||||
| `RELAYS` | Nostr relays (comma-separated) | (default set) |
|
||||
| `MODEL_PATHS_REFRESH_INTERVAL_SECONDS` | How often to refresh `/v1/models/paths` discovery data; set `0` to pause the refresh (previously discovered paths keep being served) | `600` |
|
||||
| `ENABLE_MODEL_PATHS_REFRESH` | Kill switch for the background model-path refresh (OpenRouter endpoint fan-out) | `true` |
|
||||
|
||||
Mint HTTP 429 responses create a per-mint cooldown. Operations that already hold
|
||||
Routstr's wallet mutation lock fail fast during that cooldown instead of waiting
|
||||
while blocking every other wallet mutation. Callers receive an error and may retry
|
||||
later; the current response does not include the cooldown duration.
|
||||
|
||||
### Priority
|
||||
|
||||
@@ -175,3 +248,8 @@ Manage which AI models you offer:
|
||||
- **Create aliases** — friendly names for models
|
||||
|
||||
See [Pricing](pricing.md) for per-model pricing strategies.
|
||||
|
||||
Model path discovery is refreshed in the background and exposed through
|
||||
`/v1/models/paths`. The response groups each client-visible model ID with the
|
||||
provider paths that may appear in chat-completion response metadata. Tune the
|
||||
refresh cadence with `MODEL_PATHS_REFRESH_INTERVAL_SECONDS`.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""add model paths table
|
||||
|
||||
Revision ID: 64ed5594df1f
|
||||
Revises: aa50fde387a2
|
||||
Create Date: 2026-08-02 22:26:33.280409
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "64ed5594df1f"
|
||||
down_revision = "aa50fde387a2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"model_paths",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("model_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("path", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("provider_slug", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("provider_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("endpoint_tag", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column("endpoint_name", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column("upstream_provider_id", sa.Integer(), nullable=False),
|
||||
sa.Column("updated_at", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["upstream_provider_id"], ["upstream_providers.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"model_id",
|
||||
"path",
|
||||
"upstream_provider_id",
|
||||
name="uq_model_paths_model_path_provider",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_model_paths_upstream_provider_id"),
|
||||
"model_paths",
|
||||
["upstream_provider_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f("ix_model_paths_upstream_provider_id"), table_name="model_paths")
|
||||
op.drop_table("model_paths")
|
||||
@@ -0,0 +1,62 @@
|
||||
"""add reservation release idempotency records
|
||||
|
||||
Revision ID: 7f2843d3f4e4
|
||||
Revises: fc4fa29630d2
|
||||
Create Date: 2026-07-24 02:06:06.066726
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "7f2843d3f4e4"
|
||||
down_revision = "fc4fa29630d2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"reservation_releases",
|
||||
sa.Column("id", sa.String(), nullable=False),
|
||||
sa.Column("key_hash", sa.String(), nullable=False),
|
||||
sa.Column("billing_key_hash", sa.String(), nullable=False),
|
||||
sa.Column("reserved_msats", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(), nullable=False, server_default="active"
|
||||
),
|
||||
sa.Column("created_at", sa.Integer(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_key_hash",
|
||||
"reservation_releases",
|
||||
["key_hash"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_billing_key_hash",
|
||||
"reservation_releases",
|
||||
["billing_key_hash"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_status_created_at",
|
||||
"reservation_releases",
|
||||
["status", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_status_created_at",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_billing_key_hash",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_key_hash",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_table("reservation_releases")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""repair missing fee payout checkpoint columns
|
||||
|
||||
Revision ID: 9c4d8e2f1a6b
|
||||
Revises: 7f2843d3f4e4
|
||||
Create Date: 2026-07-25 00:00:00.000000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "9c4d8e2f1a6b"
|
||||
down_revision = "7f2843d3f4e4"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Repair databases stamped past the original checkpoint migration."""
|
||||
conn = op.get_bind()
|
||||
columns = {
|
||||
column["name"] for column in sa.inspect(conn).get_columns("routstr_fees")
|
||||
}
|
||||
|
||||
if "payout_in_progress_msats" not in columns:
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column(
|
||||
"payout_in_progress_msats",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
|
||||
if "payout_started_at" not in columns:
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column("payout_started_at", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The preceding revision already expects both columns. This migration only
|
||||
# repairs schema drift, so downgrading it must preserve the expected schema.
|
||||
pass
|
||||
@@ -0,0 +1,39 @@
|
||||
"""add refund sweep claim lease
|
||||
|
||||
Revision ID: aa50fde387a2
|
||||
Revises: 9c4d8e2f1a6b
|
||||
Create Date: 2026-07-26 12:50:10.509217
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "aa50fde387a2"
|
||||
down_revision = "9c4d8e2f1a6b"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = {
|
||||
column["name"]
|
||||
for column in sa.inspect(conn).get_columns("cashu_transactions")
|
||||
}
|
||||
if "sweep_started_at" not in columns:
|
||||
op.add_column(
|
||||
"cashu_transactions",
|
||||
sa.Column("sweep_started_at", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = {
|
||||
column["name"]
|
||||
for column in sa.inspect(conn).get_columns("cashu_transactions")
|
||||
}
|
||||
if "sweep_started_at" in columns:
|
||||
op.drop_column("cashu_transactions", "sweep_started_at")
|
||||
@@ -0,0 +1,67 @@
|
||||
"""add mint url to lightning invoices
|
||||
|
||||
Revision ID: ecfa0d6e2a36
|
||||
Revises: 64ed5594df1f
|
||||
Create Date: 2026-08-02 23:53:00.037456
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "ecfa0d6e2a36"
|
||||
down_revision = "64ed5594df1f"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _resolve_backfill_mint_url(bind: sa.engine.Connection) -> str | None:
|
||||
"""Best-effort resolution of the mint that issued pre-existing invoices.
|
||||
|
||||
Order: persisted settings JSON -> PRIMARY_MINT_URL env -> first CASHU_MINTS entry.
|
||||
"""
|
||||
try:
|
||||
row = bind.execute(
|
||||
sa.text("SELECT data FROM settings ORDER BY id LIMIT 1")
|
||||
).fetchone()
|
||||
if row and row[0]:
|
||||
data = json.loads(row[0])
|
||||
mint = data.get("primary_mint") or next(
|
||||
iter(data.get("cashu_mints") or []), None
|
||||
)
|
||||
if mint:
|
||||
return str(mint)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
env_mint = os.environ.get("PRIMARY_MINT_URL", "").strip()
|
||||
if env_mint:
|
||||
return env_mint
|
||||
|
||||
cashu_mints = os.environ.get("CASHU_MINTS", "").strip()
|
||||
if cashu_mints:
|
||||
return cashu_mints.split(",")[0].strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
|
||||
)
|
||||
|
||||
bind = op.get_bind()
|
||||
backfill_mint = _resolve_backfill_mint_url(bind)
|
||||
if backfill_mint:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE lightning_invoices SET mint_url = :mint WHERE mint_url IS NULL"
|
||||
),
|
||||
{"mint": backfill_mint},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("lightning_invoices", "mint_url")
|
||||
+429
-169
@@ -3,16 +3,25 @@ import hashlib
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import case
|
||||
from sqlalchemy import case, inspect
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import col, select, update
|
||||
|
||||
from .core import get_logger
|
||||
from .core.db import ApiKey, AsyncSession, accumulate_routstr_fee
|
||||
from .core.db import (
|
||||
ApiKey,
|
||||
AsyncSession,
|
||||
ReservationRelease,
|
||||
accumulate_routstr_fee,
|
||||
create_session,
|
||||
)
|
||||
from .core.settings import settings
|
||||
from .payment.cost_calculation import (
|
||||
CostData,
|
||||
@@ -24,6 +33,7 @@ from .wallet import (
|
||||
classify_redemption_error,
|
||||
credit_balance,
|
||||
deserialize_token_from_string,
|
||||
wallet_operation_guard,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -34,10 +44,50 @@ payments_logger = get_logger("routstr.payments")
|
||||
|
||||
# Routstr platform fee constants
|
||||
ROUTSTR_FEE_PERCENT: float = 2.1
|
||||
ROUTSTR_LN_ADDRESS: str = "npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
|
||||
ROUTSTR_LN_ADDRESS: str = (
|
||||
"npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
|
||||
)
|
||||
ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
|
||||
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
|
||||
|
||||
|
||||
def _format_msat_amount(amount: int) -> str:
|
||||
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
|
||||
return f"{sats} sats ({amount} msats)"
|
||||
|
||||
|
||||
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
|
||||
return {
|
||||
"error": {
|
||||
"message": (
|
||||
f"Insufficient balance: {_format_msat_amount(required)} required "
|
||||
f"for this model; {_format_msat_amount(available)} available."
|
||||
),
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReservationSnapshot:
|
||||
release_id: str
|
||||
key_hash: str
|
||||
billing_key_hash: str
|
||||
reserved_msats: int
|
||||
|
||||
|
||||
_current_reservation: ContextVar[ReservationSnapshot | None] = ContextVar(
|
||||
"current_billing_reservation", default=None
|
||||
)
|
||||
|
||||
|
||||
def _clear_current_reservation(snapshot: ReservationSnapshot) -> None:
|
||||
current = _current_reservation.get()
|
||||
if current is not None and current.release_id == snapshot.release_id:
|
||||
_current_reservation.set(None)
|
||||
|
||||
|
||||
# TODO: implement prepaid api key (not like it was before)
|
||||
# PREPAID_API_KEY = os.environ.get("PREPAID_API_KEY", None)
|
||||
# PREPAID_BALANCE = int(os.environ.get("PREPAID_BALANCE", "0")) * 1000 # Convert to msats
|
||||
@@ -122,6 +172,29 @@ async def validate_bearer_key(
|
||||
refund_address: Optional[str] = None,
|
||||
key_expiry_time: Optional[int] = None,
|
||||
min_cost: int = 0,
|
||||
) -> ApiKey:
|
||||
if bearer_key.startswith("cashu"):
|
||||
# Acquire before the first lookup/flush so concurrent token creation
|
||||
# cannot hold SQLite write transactions while waiting to mutate proofs.
|
||||
async with wallet_operation_guard():
|
||||
return await _validate_bearer_key_locked(
|
||||
bearer_key,
|
||||
session,
|
||||
refund_address,
|
||||
key_expiry_time,
|
||||
min_cost,
|
||||
)
|
||||
return await _validate_bearer_key_locked(
|
||||
bearer_key, session, refund_address, key_expiry_time, min_cost
|
||||
)
|
||||
|
||||
|
||||
async def _validate_bearer_key_locked(
|
||||
bearer_key: str,
|
||||
session: AsyncSession,
|
||||
refund_address: Optional[str] = None,
|
||||
key_expiry_time: Optional[int] = None,
|
||||
min_cost: int = 0,
|
||||
) -> ApiKey:
|
||||
"""
|
||||
Validates the provided API key using SQLModel.
|
||||
@@ -209,13 +282,7 @@ async def validate_bearer_key(
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(min_cost, billing_key.total_balance),
|
||||
)
|
||||
|
||||
# Early check: Spending limit check (Child key limit)
|
||||
@@ -305,13 +372,9 @@ async def validate_bearer_key(
|
||||
if min_cost > 0 and existing_key.total_balance < min_cost:
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(
|
||||
min_cost, existing_key.total_balance
|
||||
),
|
||||
)
|
||||
|
||||
return existing_key
|
||||
@@ -324,11 +387,23 @@ async def validate_bearer_key(
|
||||
"has_expiry_time": bool(key_expiry_time),
|
||||
},
|
||||
)
|
||||
if token_obj.mint in settings.cashu_mints:
|
||||
if token_obj.mint == settings.primary_mint:
|
||||
if token_obj.unit != settings.primary_mint_unit:
|
||||
raise redemption_error_to_http_exception(
|
||||
ValueError(
|
||||
"Cashu token unit does not match the configured primary "
|
||||
f"mint unit: expected {settings.primary_mint_unit}, "
|
||||
f"got {token_obj.unit}"
|
||||
)
|
||||
)
|
||||
refund_currency = token_obj.unit
|
||||
refund_mint_url = settings.primary_mint
|
||||
elif token_obj.mint in settings.cashu_mints:
|
||||
refund_currency = token_obj.unit
|
||||
refund_mint_url = token_obj.mint
|
||||
else:
|
||||
refund_currency = "sat"
|
||||
# Foreign tokens are swapped into the configured primary mint.
|
||||
refund_currency = settings.primary_mint_unit
|
||||
refund_mint_url = settings.primary_mint
|
||||
|
||||
new_key = ApiKey(
|
||||
@@ -595,6 +670,16 @@ async def pay_for_request(
|
||||
},
|
||||
)
|
||||
|
||||
# Create the durable reservation identity before changing aggregate balances.
|
||||
# The row and balance updates commit together, so every reserved amount has one
|
||||
# owner that can reach exactly one terminal state.
|
||||
reservation = ReservationSnapshot(
|
||||
release_id=uuid.uuid4().hex,
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=billing_key.hashed_key,
|
||||
reserved_msats=cost_per_request,
|
||||
)
|
||||
|
||||
# Charge the base cost for the request atomically to avoid race conditions
|
||||
reserved_at_now = int(time.time())
|
||||
stmt = (
|
||||
@@ -656,22 +741,83 @@ async def pay_for_request(
|
||||
child_result = await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
|
||||
if child_result.rowcount == 0:
|
||||
# Build the error before rollback expires ORM attributes.
|
||||
limit_message = (
|
||||
f"Balance limit exceeded: {key.balance_limit} mSats limit. "
|
||||
f"{key.total_spent} already spent ({key.reserved_balance} reserved), "
|
||||
f"{cost_per_request} required for this request."
|
||||
)
|
||||
# The parent reservation update already ran in this transaction.
|
||||
# Roll it back before failover code attempts to restore the previous
|
||||
# reservation; otherwise that later commit can persist both updates.
|
||||
await session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Balance limit exceeded: {key.balance_limit} mSats limit. {key.total_spent} already spent ({key.reserved_balance} reserved), {cost_per_request} required for this request.",
|
||||
"message": limit_message,
|
||||
"type": "insufficient_quota",
|
||||
"code": "balance_limit_exceeded",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await session.commit()
|
||||
session.add(
|
||||
ReservationRelease(
|
||||
id=reservation.release_id,
|
||||
key_hash=reservation.key_hash,
|
||||
billing_key_hash=reservation.billing_key_hash,
|
||||
reserved_msats=reservation.reserved_msats,
|
||||
status="active",
|
||||
)
|
||||
)
|
||||
# Publish the identity before commit. If the commit succeeds but its
|
||||
# acknowledgement is interrupted, exact cleanup can still recover the
|
||||
# durable row. A definitely failed commit is harmless because every
|
||||
# terminal transition validates that row before touching balances.
|
||||
_current_reservation.set(reservation)
|
||||
try:
|
||||
await session.commit()
|
||||
except BaseException:
|
||||
# The database may have committed even if acknowledgement was cancelled
|
||||
# or the connection failed. Reconcile using a fresh transaction and the
|
||||
# exact durable identity; no upstream request has started yet.
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
async with create_session() as cleanup_session:
|
||||
record = await cleanup_session.get(
|
||||
ReservationRelease, reservation.release_id
|
||||
)
|
||||
if record is not None and record.status == "active":
|
||||
await _transition_reservation_to_released(
|
||||
reservation,
|
||||
cleanup_session,
|
||||
decrement_requests=True,
|
||||
idempotent_success=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to reconcile ambiguous reservation commit",
|
||||
extra={"reservation_id": reservation.release_id},
|
||||
)
|
||||
finally:
|
||||
_clear_current_reservation(reservation)
|
||||
raise
|
||||
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
try:
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
except Exception:
|
||||
# The reservation transaction is already committed and durable. Logging
|
||||
# refresh failures must not make the caller treat it as unreserved.
|
||||
logger.exception(
|
||||
"Reservation committed but post-commit refresh failed",
|
||||
extra={"reservation_id": reservation.release_id},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Payment processed successfully",
|
||||
@@ -701,81 +847,185 @@ async def pay_for_request(
|
||||
|
||||
|
||||
async def revert_pay_for_request(
|
||||
key: ApiKey, session: AsyncSession, cost_per_request: int
|
||||
key: ApiKey,
|
||||
session: AsyncSession,
|
||||
cost_per_request: int,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> bool:
|
||||
"""Revert a previously reserved payment. Returns True if revert succeeded,
|
||||
False if the reservation was already released (prevents negative reserved_balance)."""
|
||||
billing_key = await get_billing_key(key, session)
|
||||
|
||||
# Keep reserved_at while other reservations remain
|
||||
cleared_reserved_at = case(
|
||||
(col(ApiKey.reserved_balance) - cost_per_request > 0, col(ApiKey.reserved_at)),
|
||||
else_=None,
|
||||
"""Revert the current request's durable reservation exactly once."""
|
||||
snapshot = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, snapshot, session, require_active=False)
|
||||
if cost_per_request != snapshot.reserved_msats:
|
||||
return False
|
||||
return await _transition_reservation_to_released(
|
||||
snapshot,
|
||||
session,
|
||||
decrement_requests=True,
|
||||
idempotent_success=False,
|
||||
)
|
||||
|
||||
stmt = (
|
||||
|
||||
async def _validate_reservation_snapshot(
|
||||
key: ApiKey,
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
require_active: bool = True,
|
||||
) -> None:
|
||||
"""Reject cross-request or forged reservation handles before any mutation."""
|
||||
state = inspect(key)
|
||||
identity = state.identity if state is not None else None
|
||||
key_hash = str(identity[0]) if identity else key.__dict__.get("hashed_key")
|
||||
if snapshot.key_hash != key_hash:
|
||||
raise RuntimeError("Billing reservation does not belong to this key")
|
||||
|
||||
persisted_key = await session.get(ApiKey, snapshot.key_hash)
|
||||
if persisted_key is None:
|
||||
raise RuntimeError("Billing reservation key no longer exists")
|
||||
expected_billing_hash = persisted_key.parent_key_hash or persisted_key.hashed_key
|
||||
if snapshot.billing_key_hash != expected_billing_hash:
|
||||
raise RuntimeError("Billing reservation does not belong to this billing key")
|
||||
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
if (
|
||||
record is None
|
||||
or (require_active and record.status != "active")
|
||||
or record.key_hash != snapshot.key_hash
|
||||
or record.billing_key_hash != snapshot.billing_key_hash
|
||||
or record.reserved_msats != snapshot.reserved_msats
|
||||
):
|
||||
raise RuntimeError("Billing reservation record does not match the request")
|
||||
|
||||
|
||||
async def get_reservation_snapshot(
|
||||
key: ApiKey, session: AsyncSession
|
||||
) -> ReservationSnapshot:
|
||||
"""Return the durable reservation created for the current request."""
|
||||
snapshot = _current_reservation.get()
|
||||
if snapshot is None:
|
||||
raise RuntimeError("No billing reservation is associated with this request")
|
||||
await _validate_reservation_snapshot(key, snapshot, session)
|
||||
return snapshot
|
||||
|
||||
|
||||
async def _transition_reservation_to_released(
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
decrement_requests: bool,
|
||||
idempotent_success: bool,
|
||||
) -> bool:
|
||||
transition = (
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == snapshot.release_id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
|
||||
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
|
||||
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
|
||||
.values(status="released")
|
||||
)
|
||||
transition_result = await session.exec(transition) # type: ignore[call-overload]
|
||||
if transition_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
existing = await session.get(ReservationRelease, snapshot.release_id)
|
||||
return bool(
|
||||
idempotent_success
|
||||
and existing is not None
|
||||
and existing.status == "released"
|
||||
and existing.key_hash == snapshot.key_hash
|
||||
and existing.billing_key_hash == snapshot.billing_key_hash
|
||||
and existing.reserved_msats == snapshot.reserved_msats
|
||||
)
|
||||
|
||||
values: dict[str, object] = {
|
||||
"reserved_balance": col(ApiKey.reserved_balance) - snapshot.reserved_msats,
|
||||
"reserved_at": case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) - snapshot.reserved_msats > 0,
|
||||
col(ApiKey.reserved_at),
|
||||
),
|
||||
else_=None,
|
||||
),
|
||||
}
|
||||
if decrement_requests:
|
||||
values["total_requests"] = col(ApiKey.total_requests) - 1
|
||||
|
||||
release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= cost_per_request)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
|
||||
reserved_at=cleared_reserved_at,
|
||||
total_requests=col(ApiKey.total_requests) - 1,
|
||||
)
|
||||
.where(col(ApiKey.hashed_key) == snapshot.billing_key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
result = await session.exec(release_stmt) # type: ignore[call-overload]
|
||||
if result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
|
||||
# Also decrement total_requests and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_stmt = (
|
||||
if snapshot.billing_key_hash != snapshot.key_hash:
|
||||
child_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= cost_per_request)
|
||||
.values(
|
||||
total_requests=col(ApiKey.total_requests) - 1,
|
||||
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
|
||||
reserved_at=cleared_reserved_at,
|
||||
)
|
||||
.where(col(ApiKey.hashed_key) == snapshot.key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
child_result = await session.exec( # type: ignore[call-overload]
|
||||
child_release_stmt
|
||||
)
|
||||
if child_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
await session.commit()
|
||||
if result.rowcount == 0:
|
||||
logger.warning(
|
||||
"Revert skipped - reservation already released (no-op to prevent negative reserved_balance)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"cost_to_revert": cost_per_request,
|
||||
"current_reserved_balance": billing_key.reserved_balance,
|
||||
},
|
||||
)
|
||||
return False
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
payments_logger.info(
|
||||
"REVERT",
|
||||
extra={
|
||||
"event": "revert",
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"cost_reverted": cost_per_request,
|
||||
"balance": billing_key.balance,
|
||||
"reserved_balance": billing_key.reserved_balance,
|
||||
},
|
||||
)
|
||||
_clear_current_reservation(snapshot)
|
||||
return True
|
||||
|
||||
|
||||
async def release_reservation(
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
reserved_msats: int,
|
||||
) -> bool:
|
||||
"""Release one durable reservation exactly once without charging."""
|
||||
if reserved_msats <= 0 or reserved_msats != snapshot.reserved_msats:
|
||||
return False
|
||||
return await _transition_reservation_to_released(
|
||||
snapshot,
|
||||
session,
|
||||
decrement_requests=False,
|
||||
idempotent_success=True,
|
||||
)
|
||||
|
||||
|
||||
async def _claim_reservation_for_charge(
|
||||
snapshot: ReservationSnapshot, session: AsyncSession
|
||||
) -> bool:
|
||||
"""Claim an active reservation in the caller's charge transaction."""
|
||||
statement = (
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == snapshot.release_id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
|
||||
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
|
||||
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
|
||||
.values(status="charged")
|
||||
)
|
||||
result = await session.exec(statement) # type: ignore[call-overload]
|
||||
if result.rowcount == 1:
|
||||
_clear_current_reservation(snapshot)
|
||||
return True
|
||||
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
|
||||
async def adjust_payment_for_tokens(
|
||||
key: ApiKey,
|
||||
response_data: dict,
|
||||
session: AsyncSession,
|
||||
deducted_max_cost: int,
|
||||
model_obj: "Model | None",
|
||||
provider_fee: float | None,
|
||||
model_obj: "Model | None" = None,
|
||||
provider_fee: float | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Adjusts the payment based on token usage in the response.
|
||||
@@ -790,6 +1040,13 @@ async def adjust_payment_for_tokens(
|
||||
``calculate_cost``.
|
||||
"""
|
||||
billing_key = await get_billing_key(key, session)
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(
|
||||
key, reservation, session, require_active=False
|
||||
)
|
||||
# The persisted amount is authoritative if request-level minimum pricing
|
||||
# changed the caller's original estimate.
|
||||
deducted_max_cost = reservation.reserved_msats
|
||||
model = response_data.get("model", "unknown")
|
||||
|
||||
logger.debug(
|
||||
@@ -805,50 +1062,21 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
async def release_reservation_only() -> None:
|
||||
"""Fallback to release reservation without charging when main update fails."""
|
||||
"""Fallback to release this request's reservation without charging."""
|
||||
try:
|
||||
release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost
|
||||
)
|
||||
released = await release_reservation(
|
||||
reservation, session, reservation.reserved_msats
|
||||
)
|
||||
logger.warning(
|
||||
"Released reservation without charging (fallback)"
|
||||
if released
|
||||
else "Reservation was already finalized; fallback skipped",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
result = await session.exec(release_stmt) # type: ignore[call-overload]
|
||||
|
||||
# Also release on child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance)
|
||||
- deducted_max_cost
|
||||
)
|
||||
)
|
||||
await session.exec(child_release_stmt) # type: ignore[call-overload]
|
||||
|
||||
await session.commit()
|
||||
if result.rowcount == 0: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"Release reservation skipped - already released (no-op to prevent negative reserved_balance)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Released reservation without charging (fallback)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to release reservation in fallback",
|
||||
@@ -870,9 +1098,17 @@ async def adjust_payment_for_tokens(
|
||||
extra={"error": str(e), "fee_msats": fee_msats},
|
||||
)
|
||||
|
||||
match await calculate_cost(
|
||||
calculated_cost = await calculate_cost(
|
||||
response_data, deducted_max_cost, model_obj, provider_fee
|
||||
):
|
||||
)
|
||||
if not isinstance(calculated_cost, CostDataError):
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
# A prior charge or release already owns this reservation. Returning
|
||||
# the calculated metadata is safe; the aggregate balances must not
|
||||
# be modified a second time.
|
||||
return calculated_cost.dict()
|
||||
|
||||
match calculated_cost:
|
||||
case MaxCostData() as cost:
|
||||
logger.debug(
|
||||
"Using max cost data (no token adjustment)",
|
||||
@@ -900,8 +1136,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -919,8 +1157,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1030,8 +1270,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
exact_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -1049,8 +1291,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_exact_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1089,31 +1333,45 @@ async def adjust_payment_for_tokens(
|
||||
|
||||
# actual cost exceeded discounted reservation (due to tolerance_percentage)
|
||||
if cost_difference > 0:
|
||||
# Always release the reservation and charge min(actual_cost, balance).
|
||||
# CASE expressions keep this atomic and safe even when the
|
||||
# stale-reservation sweeper has already released the reservation.
|
||||
chargeable = case(
|
||||
(col(ApiKey.balance) >= total_cost_msats, total_cost_msats),
|
||||
else_=col(ApiKey.balance),
|
||||
)
|
||||
overrun_safe_reserved = case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
finalize_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
balance=col(ApiKey.balance) - chargeable,
|
||||
total_spent=col(ApiKey.total_spent) + chargeable,
|
||||
# Lock the billing row so the parent and child record the same
|
||||
# database-determined charge under concurrent finalizations.
|
||||
actual_charge_msats = 0
|
||||
for attempt in range(5):
|
||||
locked_billing_key = (
|
||||
await session.exec(
|
||||
select(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).one()
|
||||
observed_balance = locked_billing_key.balance
|
||||
actual_charge_msats = min(observed_balance, total_cost_msats)
|
||||
overrun_safe_reserved = case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
)
|
||||
await session.exec(finalize_stmt) # type: ignore[call-overload]
|
||||
finalize_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.balance) == observed_balance)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
balance=col(ApiKey.balance) - actual_charge_msats,
|
||||
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
|
||||
)
|
||||
)
|
||||
if finalize_result.rowcount == 1:
|
||||
break
|
||||
await session.rollback()
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return cost.dict()
|
||||
else:
|
||||
await session.rollback()
|
||||
raise RuntimeError("Could not atomically finalize cost overrun")
|
||||
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_stmt = (
|
||||
@@ -1121,7 +1379,7 @@ async def adjust_payment_for_tokens(
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
total_spent=col(ApiKey.total_spent) + min(billing_key.balance, total_cost_msats),
|
||||
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
|
||||
)
|
||||
)
|
||||
await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
@@ -1131,18 +1389,18 @@ async def adjust_payment_for_tokens(
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
cost.total_msats = total_cost_msats
|
||||
cost.total_msats = actual_charge_msats
|
||||
logger.info(
|
||||
"Finalized payment with additional charge",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"charged_amount": total_cost_msats,
|
||||
"charged_amount": actual_charge_msats,
|
||||
"new_balance": billing_key.balance,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
await _accumulate_fee(total_cost_msats)
|
||||
await _accumulate_fee(actual_charge_msats)
|
||||
payments_logger.info(
|
||||
"FINALIZE",
|
||||
extra={
|
||||
@@ -1151,7 +1409,7 @@ async def adjust_payment_for_tokens(
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"model": model,
|
||||
"cost_reserved": deducted_max_cost,
|
||||
"cost_charged": total_cost_msats,
|
||||
"cost_charged": actual_charge_msats,
|
||||
"input_tokens": cost.input_tokens,
|
||||
"output_tokens": cost.output_tokens,
|
||||
"balance": billing_key.balance,
|
||||
@@ -1191,8 +1449,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
refund_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -1210,8 +1470,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_refund_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1386,9 +1648,7 @@ async def periodic_dead_key_prune() -> None:
|
||||
|
||||
try:
|
||||
async with create_session() as session:
|
||||
await prune_dead_api_keys(
|
||||
session, settings.dead_key_min_age_seconds
|
||||
)
|
||||
await prune_dead_api_keys(session, settings.dead_key_min_age_seconds)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
|
||||
+192
-39
@@ -7,7 +7,7 @@ from typing import Annotated, NoReturn
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlmodel import col, or_, select, update
|
||||
from sqlmodel import col, select, update
|
||||
|
||||
from .auth import get_billing_key, validate_bearer_key
|
||||
from .core.db import (
|
||||
@@ -15,6 +15,7 @@ from .core.db import (
|
||||
AsyncSession,
|
||||
CashuTransaction,
|
||||
get_session,
|
||||
release_stale_reservations,
|
||||
)
|
||||
from .core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
@@ -22,6 +23,7 @@ from .core.db import (
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .lightning import lightning_router
|
||||
from .payment.lnurl import MeltOutcomeAmbiguousError
|
||||
from .wallet import (
|
||||
classify_redemption_error,
|
||||
credit_balance,
|
||||
@@ -29,6 +31,7 @@ from .wallet import (
|
||||
recieve_token,
|
||||
send_to_lnurl,
|
||||
send_token,
|
||||
token_mint_url,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
@@ -108,13 +111,19 @@ async def account_info(
|
||||
# Note: validate_bearer_key already supports refund_address and key_expiry_time params
|
||||
|
||||
|
||||
@router.get("/create")
|
||||
async def create_balance(
|
||||
class BalanceCreateRequest(BaseModel):
|
||||
initial_balance_token: str
|
||||
balance_limit: int | None = None
|
||||
balance_limit_reset: str | None = None
|
||||
validity_date: int | None = None
|
||||
|
||||
|
||||
async def _create_balance(
|
||||
initial_balance_token: str,
|
||||
balance_limit: int | None = None,
|
||||
balance_limit_reset: str | None = None,
|
||||
validity_date: int | None = None,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
balance_limit: int | None,
|
||||
balance_limit_reset: str | None,
|
||||
validity_date: int | None,
|
||||
session: AsyncSession,
|
||||
) -> dict:
|
||||
key = await validate_bearer_key(initial_balance_token, session)
|
||||
|
||||
@@ -134,6 +143,37 @@ async def create_balance(
|
||||
}
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def create_balance_from_body(
|
||||
payload: BalanceCreateRequest,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict:
|
||||
return await _create_balance(
|
||||
payload.initial_balance_token,
|
||||
payload.balance_limit,
|
||||
payload.balance_limit_reset,
|
||||
payload.validity_date,
|
||||
session,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/create")
|
||||
async def create_balance(
|
||||
initial_balance_token: str,
|
||||
balance_limit: int | None = None,
|
||||
balance_limit_reset: str | None = None,
|
||||
validity_date: int | None = None,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict:
|
||||
return await _create_balance(
|
||||
initial_balance_token,
|
||||
balance_limit,
|
||||
balance_limit_reset,
|
||||
validity_date,
|
||||
session,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/info")
|
||||
async def wallet_info(
|
||||
key: ApiKey = Depends(get_key_from_header),
|
||||
@@ -146,6 +186,17 @@ class TopupRequest(BaseModel):
|
||||
cashu_token: str
|
||||
|
||||
|
||||
def _error_chain(error: BaseException) -> list[dict[str, str]]:
|
||||
chain: list[dict[str, str]] = []
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
chain.append({"type": type(current).__name__, "message": str(current)})
|
||||
current = current.__cause__ or current.__context__
|
||||
return chain
|
||||
|
||||
|
||||
@router.post("/topup")
|
||||
async def topup_wallet_endpoint(
|
||||
cashu_token: str | None = None,
|
||||
@@ -163,6 +214,18 @@ async def topup_wallet_endpoint(
|
||||
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
|
||||
if len(cashu_token) < 10 or "cashu" not in cashu_token:
|
||||
raise HTTPException(status_code=400, detail="Invalid token format")
|
||||
|
||||
source_mint = token_mint_url(cashu_token, "unknown")
|
||||
logger.info(
|
||||
"Cashu wallet top-up started",
|
||||
extra={
|
||||
"event": "cashu_topup_started",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
try:
|
||||
amount_msats = await credit_balance(cashu_token, billing_key, session)
|
||||
except Exception as e:
|
||||
@@ -171,12 +234,41 @@ async def topup_wallet_endpoint(
|
||||
classified = classify_redemption_error(e)
|
||||
if classified is None:
|
||||
logger.error(
|
||||
"topup_wallet_endpoint: unhandled error",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
"Cashu wallet top-up failed with an unhandled error",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
_type, status_code, message, _code = classified
|
||||
error_type, status_code, message, error_code = classified
|
||||
logger.warning(
|
||||
"Cashu wallet top-up failed",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"status_code": status_code,
|
||||
"error_type": error_type,
|
||||
"error_code": error_code,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=message)
|
||||
|
||||
logger.info(
|
||||
"Cashu wallet top-up completed",
|
||||
extra={
|
||||
"event": "cashu_topup_completed",
|
||||
"source_mint": source_mint,
|
||||
"credited_msats": amount_msats,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
return {"msats": amount_msats}
|
||||
|
||||
|
||||
@@ -221,8 +313,42 @@ async def _lookup_key_no_create(
|
||||
return None
|
||||
|
||||
|
||||
async def _get_persisted_api_key_refund(
|
||||
key: ApiKey, session: AsyncSession
|
||||
) -> dict[str, str] | None:
|
||||
result = await session.exec(
|
||||
select(CashuTransaction)
|
||||
.where(
|
||||
CashuTransaction.api_key_hashed_key == key.hashed_key,
|
||||
CashuTransaction.type == "out",
|
||||
CashuTransaction.source == "apikey",
|
||||
)
|
||||
.order_by(col(CashuTransaction.created_at).desc())
|
||||
)
|
||||
refund = result.first()
|
||||
if refund is None:
|
||||
return None
|
||||
if refund.swept:
|
||||
raise HTTPException(status_code=410, detail="Refund has been swept")
|
||||
|
||||
refund.collected = True
|
||||
session.add(refund)
|
||||
await session.commit()
|
||||
|
||||
persisted = {"token": refund.token}
|
||||
if refund.unit == "sat":
|
||||
persisted["sats"] = str(refund.amount)
|
||||
else:
|
||||
persisted["msats"] = str(refund.amount)
|
||||
return persisted
|
||||
|
||||
|
||||
async def _restore_balance(
|
||||
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
|
||||
session: AsyncSession,
|
||||
hashed_key: str,
|
||||
balance: int,
|
||||
reserved_balance: int,
|
||||
mint_url: str,
|
||||
) -> None:
|
||||
"""Restore balance after a failed refund mint attempt."""
|
||||
restore_stmt = (
|
||||
@@ -237,7 +363,11 @@ async def _restore_balance(
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"refund_wallet_endpoint: balance restored after mint failure",
|
||||
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
|
||||
extra={
|
||||
"hashed_key": hashed_key,
|
||||
"restored_balance": balance,
|
||||
"mint_url": mint_url,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -315,6 +445,8 @@ async def refund_wallet_endpoint(
|
||||
if key.total_balance <= 0:
|
||||
if cached := await _refund_cache_get(bearer_value):
|
||||
return cached
|
||||
if persisted := await _get_persisted_api_key_refund(key, session):
|
||||
return persisted
|
||||
|
||||
if key.parent_key_hash:
|
||||
raise HTTPException(
|
||||
@@ -323,30 +455,19 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
|
||||
if key.reserved_balance > 0:
|
||||
# Release the reservation if it is stale
|
||||
cutoff = int(time.time()) - settings.stale_reservation_timeout_seconds
|
||||
stale_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) > 0)
|
||||
.where(
|
||||
or_(
|
||||
col(ApiKey.reserved_at).is_(None),
|
||||
col(ApiKey.reserved_at) < cutoff,
|
||||
)
|
||||
)
|
||||
.values(reserved_balance=0, reserved_at=None)
|
||||
# Release only durable reservations old enough to be stale. A newer
|
||||
# request on the same aggregate balance must remain reserved.
|
||||
await release_stale_reservations(
|
||||
session,
|
||||
settings.stale_reservation_timeout_seconds,
|
||||
key_hash=key.hashed_key,
|
||||
)
|
||||
stale_result = await session.exec(stale_release_stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
|
||||
if stale_result.rowcount == 0:
|
||||
await session.refresh(key)
|
||||
if key.reserved_balance > 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot refund key. There are ongoing requests for this api key.",
|
||||
)
|
||||
|
||||
await session.refresh(key)
|
||||
logger.warning(
|
||||
"refund_wallet_endpoint: released stale reservation before refund",
|
||||
extra={
|
||||
@@ -391,15 +512,14 @@ async def refund_wallet_endpoint(
|
||||
detail="Balance changed concurrently. Please retry the refund.",
|
||||
)
|
||||
|
||||
# --- MINT: balance is locked at zero, safe to create the refund token ---
|
||||
# Proofs from untrusted mints are swapped to primary_mint on receive.
|
||||
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
|
||||
# The balance is locked at zero, so it is safe to create the refund token.
|
||||
effective_refund_mint = (
|
||||
key.refund_mint_url
|
||||
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
|
||||
else settings.primary_mint
|
||||
)
|
||||
try:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
if key.refund_address:
|
||||
await send_to_lnurl(
|
||||
remaining_balance,
|
||||
@@ -409,10 +529,10 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
result = {"recipient": key.refund_address}
|
||||
else:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
token = await send_token(
|
||||
remaining_balance, refund_currency, effective_refund_mint
|
||||
)
|
||||
effective_refund_mint = token_mint_url(token, effective_refund_mint)
|
||||
result = {"token": token}
|
||||
|
||||
if key.refund_currency == "sat":
|
||||
@@ -431,13 +551,47 @@ async def refund_wallet_endpoint(
|
||||
},
|
||||
)
|
||||
|
||||
except MeltOutcomeAmbiguousError as e:
|
||||
# The melt was dispatched and may still settle. Restoring the balance
|
||||
# here would let the same debit be paid out twice; keep the debit and
|
||||
# leave the outcome to reconciliation.
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: melt outcome ambiguous; balance withheld "
|
||||
"pending reconciliation",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"hashed_key": key.hashed_key,
|
||||
"remaining_balance": remaining_balance,
|
||||
"refund_currency": key.refund_currency,
|
||||
"refund_mint_url": key.refund_mint_url,
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=(
|
||||
"Refund was dispatched but its outcome is unconfirmed; the "
|
||||
"balance is withheld until reconciliation completes"
|
||||
),
|
||||
)
|
||||
except HTTPException:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
error_msg = str(e)
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: mint/send failed",
|
||||
@@ -464,7 +618,7 @@ async def refund_wallet_endpoint(
|
||||
token=result["token"],
|
||||
amount=remaining_balance,
|
||||
unit=key.refund_currency or "sat",
|
||||
mint_url=key.refund_mint_url,
|
||||
mint_url=effective_refund_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="apikey",
|
||||
@@ -658,7 +812,6 @@ async def reset_child_key_spent(
|
||||
return {"success": True, "message": "Child key balance reset successfully."}
|
||||
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/{path:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE"],
|
||||
|
||||
+222
-30
@@ -13,13 +13,7 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..payment.models import _row_to_model, list_models
|
||||
from ..proxy import refresh_model_maps, reinitialize_upstreams
|
||||
from ..wallet import (
|
||||
fetch_all_balances,
|
||||
get_proofs_per_mint_and_unit,
|
||||
get_wallet,
|
||||
send_token,
|
||||
slow_filter_spend_proofs,
|
||||
)
|
||||
from ..wallet import fetch_all_balances, send_token, token_mint_url
|
||||
from . import vault
|
||||
from .db import (
|
||||
ApiKey,
|
||||
@@ -51,6 +45,13 @@ ADMIN_SESSION_DURATION = 3600
|
||||
MAX_USAGE_ANALYTICS_HOURS = 365 * 24
|
||||
|
||||
|
||||
async def _refresh_provider_model_paths(upstream_provider_id: int) -> None:
|
||||
"""Queue discovery sync without blocking the committed admin mutation."""
|
||||
from ..upstream.model_paths import schedule_model_paths_refresh_for_provider
|
||||
|
||||
await schedule_model_paths_refresh_for_provider(upstream_provider_id)
|
||||
|
||||
|
||||
async def require_admin_api(request: Request) -> None:
|
||||
auth_header = request.headers.get("Authorization")
|
||||
if not auth_header or not auth_header.startswith("Bearer "):
|
||||
@@ -435,37 +436,31 @@ class WithdrawRequest(BaseModel):
|
||||
async def withdraw(
|
||||
request: Request, withdraw_request: WithdrawRequest
|
||||
) -> dict[str, str]:
|
||||
# Get wallet and check balance
|
||||
from .settings import settings as global_settings
|
||||
|
||||
effective_mint = withdraw_request.mint_url or global_settings.primary_mint
|
||||
wallet = await get_wallet(effective_mint, withdraw_request.unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet,
|
||||
effective_mint,
|
||||
withdraw_request.unit,
|
||||
not_reserved=True,
|
||||
)
|
||||
proofs = await slow_filter_spend_proofs(proofs, wallet)
|
||||
current_balance = sum(proof.amount for proof in proofs)
|
||||
|
||||
if withdraw_request.amount <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Withdrawal amount must be positive"
|
||||
)
|
||||
|
||||
if withdraw_request.amount > current_balance:
|
||||
raise HTTPException(status_code=400, detail="Insufficient wallet balance")
|
||||
|
||||
token = await send_token(
|
||||
withdraw_request.amount, withdraw_request.unit, effective_mint
|
||||
)
|
||||
try:
|
||||
token = await send_token(
|
||||
withdraw_request.amount, withdraw_request.unit, effective_mint
|
||||
)
|
||||
except ValueError as error:
|
||||
if not str(error).startswith("No trusted mint has "):
|
||||
raise
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Insufficient wallet balance"
|
||||
) from error
|
||||
actual_mint = token_mint_url(token, effective_mint)
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=withdraw_request.amount,
|
||||
unit=withdraw_request.unit,
|
||||
mint_url=effective_mint,
|
||||
mint_url=actual_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
@@ -476,10 +471,10 @@ async def withdraw(
|
||||
extra={
|
||||
"amount": withdraw_request.amount,
|
||||
"unit": withdraw_request.unit,
|
||||
"mint_url": effective_mint,
|
||||
"mint_url": actual_mint,
|
||||
},
|
||||
)
|
||||
return {"token": token}
|
||||
return {"token": token, "mint_url": actual_mint}
|
||||
|
||||
|
||||
class ModelCreate(BaseModel):
|
||||
@@ -579,6 +574,7 @@ async def upsert_provider_model(
|
||||
await session.refresh(row)
|
||||
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return _row_to_model(
|
||||
row, apply_provider_fee=True, provider_fee=provider.provider_fee
|
||||
).dict() # type: ignore
|
||||
@@ -633,6 +629,7 @@ async def delete_provider_model(provider_id: str, model_id: str) -> dict[str, ob
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {"ok": True, "deleted_id": model_id}
|
||||
|
||||
|
||||
@@ -652,6 +649,7 @@ async def delete_all_provider_models(provider_id: str) -> dict[str, object]:
|
||||
await session.delete(row) # type: ignore
|
||||
await session.commit()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {"ok": True, "deleted": len(rows)}
|
||||
|
||||
|
||||
@@ -743,6 +741,7 @@ async def batch_override_provider_models(
|
||||
await session.commit()
|
||||
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {
|
||||
"ok": True,
|
||||
"count": overridden_count,
|
||||
@@ -863,6 +862,33 @@ class UpstreamProviderUpdateBySlug(BaseModel):
|
||||
provider_settings: dict | None = None
|
||||
|
||||
|
||||
async def _active_ppq_claim_in_session(session: AsyncSession, provider_id: int) -> bool:
|
||||
"""Check for an active claim inside the caller's transaction.
|
||||
|
||||
Must share the transaction of whatever destructive write it is guarding —
|
||||
a check in its own session leaves a window for a worker to create the
|
||||
claim between the check and the commit.
|
||||
"""
|
||||
from ..upstream.auto_topup import _ppq_state_id_for_provider
|
||||
|
||||
claim = await session.get(CashuTransaction, _ppq_state_id_for_provider(provider_id))
|
||||
return claim is not None and not claim.collected and not claim.swept
|
||||
|
||||
|
||||
def _require_valid_ppq_auto_topup(
|
||||
provider_type: str, settings: dict | None
|
||||
) -> None:
|
||||
"""Reject PPQ auto top-up settings the worker would later refuse."""
|
||||
if provider_type != "ppqai":
|
||||
return
|
||||
|
||||
from ..upstream.auto_topup import validate_ppq_auto_topup_settings
|
||||
|
||||
problem = validate_ppq_auto_topup_settings(settings)
|
||||
if problem is not None:
|
||||
raise HTTPException(status_code=400, detail=problem)
|
||||
|
||||
|
||||
async def _apply_provider_update(
|
||||
session: AsyncSession,
|
||||
provider: UpstreamProviderRow,
|
||||
@@ -874,6 +900,29 @@ async def _apply_provider_update(
|
||||
await _ensure_unique_slug(session, validated, exclude_id=provider.id)
|
||||
provider.slug = validated
|
||||
|
||||
provider_type_changed = (
|
||||
payload.provider_type is not None
|
||||
and payload.provider_type != provider.provider_type
|
||||
)
|
||||
ppq_type_changed = provider_type_changed and (
|
||||
provider.provider_type == "ppqai" or payload.provider_type == "ppqai"
|
||||
)
|
||||
if (
|
||||
provider_type_changed
|
||||
and provider.provider_type == "ppqai"
|
||||
and provider.id is not None
|
||||
and await _active_ppq_claim_in_session(session, provider.id)
|
||||
):
|
||||
# Changing the type would orphan the claim: the PPQ endpoints refuse
|
||||
# non-ppqai providers, so nobody could ever inspect or release it.
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"This provider has an active PPQ auto top-up claim. Release "
|
||||
"it before changing the provider type"
|
||||
),
|
||||
)
|
||||
|
||||
if payload.provider_type is not None:
|
||||
provider.provider_type = payload.provider_type
|
||||
if payload.base_url is not None:
|
||||
@@ -886,6 +935,41 @@ async def _apply_provider_update(
|
||||
provider.enabled = payload.enabled
|
||||
if payload.provider_fee is not None:
|
||||
provider.provider_fee = payload.provider_fee
|
||||
|
||||
# Auto-top-up fields have provider-specific units and meaning. Reusing
|
||||
# enabled Routstr settings for PPQ (or vice versa) can silently reinterpret
|
||||
# sats as USD, so a type change must provide settings for the new type.
|
||||
if (
|
||||
ppq_type_changed
|
||||
and payload.provider_settings is None
|
||||
and provider.provider_settings
|
||||
):
|
||||
try:
|
||||
stored_settings = json.loads(provider.provider_settings)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
stored_settings = None
|
||||
if isinstance(stored_settings, dict) and stored_settings.get("auto_topup"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"Changing provider type requires explicit auto-top-up "
|
||||
"settings because the units are provider-specific"
|
||||
),
|
||||
)
|
||||
|
||||
# Validate against the effective type and effective settings.
|
||||
effective_settings = payload.provider_settings
|
||||
if effective_settings is None and payload.provider_type is not None:
|
||||
try:
|
||||
effective_settings = (
|
||||
json.loads(provider.provider_settings)
|
||||
if provider.provider_settings
|
||||
else None
|
||||
)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
effective_settings = None
|
||||
if effective_settings is not None:
|
||||
_require_valid_ppq_auto_topup(provider.provider_type, effective_settings)
|
||||
if payload.provider_settings is not None:
|
||||
provider.provider_settings = json.dumps(payload.provider_settings)
|
||||
|
||||
@@ -925,6 +1009,10 @@ async def create_upstream_provider(
|
||||
else:
|
||||
slug = await allocate_unique_provider_slug(session, payload.provider_type)
|
||||
|
||||
_require_valid_ppq_auto_topup(
|
||||
payload.provider_type, payload.provider_settings
|
||||
)
|
||||
|
||||
provider = UpstreamProviderRow(
|
||||
slug=slug,
|
||||
provider_type=payload.provider_type,
|
||||
@@ -943,6 +1031,7 @@ async def create_upstream_provider(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
@@ -968,6 +1057,7 @@ async def update_upstream_provider(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
@@ -1003,6 +1093,7 @@ async def update_upstream_provider_by_slug(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
@@ -1013,6 +1104,25 @@ async def delete_upstream_provider(provider_id: str) -> dict[str, object]:
|
||||
async with create_session() as session:
|
||||
provider = await _get_upstream_provider_by_ref(session, provider_id)
|
||||
deleted_id = _provider_pk(provider)
|
||||
|
||||
# Checked inside the delete transaction: the worker's claim creation
|
||||
# re-reads the provider inside its own transaction, so these two
|
||||
# writes serialise — either the claim lands first and this 409s, or
|
||||
# the delete lands first and the worker refuses to claim.
|
||||
if provider.provider_type == "ppqai" and await _active_ppq_claim_in_session(
|
||||
session, deleted_id
|
||||
):
|
||||
# Deleting now would orphan the claim and any funds it tracks:
|
||||
# the PPQ endpoints 404 without the provider row, so the claim
|
||||
# could never again be inspected or released.
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"This provider has an active PPQ auto top-up claim. "
|
||||
"Resolve and release it before deleting the provider"
|
||||
),
|
||||
)
|
||||
|
||||
await session.delete(provider)
|
||||
await session.commit()
|
||||
await reinitialize_upstreams()
|
||||
@@ -1621,6 +1731,78 @@ async def get_log_dates_api(request: Request) -> dict[str, object]:
|
||||
return {"dates": dates}
|
||||
|
||||
|
||||
_PPQ_RELEASE_ERRORS = {
|
||||
"no_active_claim": "No active PPQ claim to release",
|
||||
"stale_state": ("The claim changed since it was reviewed; reload and check again"),
|
||||
"payment_in_flight": (
|
||||
"A Lightning payment is still in flight for this claim. Wait for it to "
|
||||
"finish or expire before releasing"
|
||||
),
|
||||
"claim_changed": (
|
||||
"The claim changed while the release was being applied; reload and check again"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class ReleasePPQAutoTopupRequest(BaseModel):
|
||||
confirmed_safe_to_retry: bool
|
||||
# Echoes the state_token the admin reviewed — the claim's full versioned
|
||||
# state, not just its operation id. Any change since the review (a new
|
||||
# attempt, a phase change, a renewed lease) fails the match, so the
|
||||
# release cannot land on a state the admin never saw.
|
||||
state_token: str | None = None
|
||||
|
||||
|
||||
async def _require_ppq_provider(provider_id: int) -> UpstreamProviderRow:
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, provider_id)
|
||||
if provider is None:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
if provider.provider_type != "ppqai":
|
||||
raise HTTPException(status_code=400, detail="Provider is not PPQ")
|
||||
return provider
|
||||
|
||||
|
||||
@admin_router.get(
|
||||
"/api/upstream-providers/{provider_id}/ppq-auto-topup",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def get_ppq_auto_topup_api(provider_id: int) -> dict[str, object]:
|
||||
await _require_ppq_provider(provider_id)
|
||||
from ..upstream.auto_topup import get_ppq_auto_topup_state
|
||||
|
||||
return {"ok": True, **await get_ppq_auto_topup_state(provider_id)}
|
||||
|
||||
|
||||
@admin_router.post(
|
||||
"/api/upstream-providers/{provider_id}/ppq-auto-topup/release",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def release_ppq_auto_topup_api(
|
||||
provider_id: int, payload: ReleasePPQAutoTopupRequest
|
||||
) -> dict[str, object]:
|
||||
await _require_ppq_provider(provider_id)
|
||||
if not payload.confirmed_safe_to_retry:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Confirm the Lightning payment outcome is safe before releasing",
|
||||
)
|
||||
|
||||
from ..upstream.auto_topup import release_ppq_auto_topup_state
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(
|
||||
provider_id, state_token=payload.state_token
|
||||
)
|
||||
if not outcome.released:
|
||||
raise HTTPException(status_code=409, detail=_PPQ_RELEASE_ERRORS[outcome.reason])
|
||||
|
||||
logger.warning(
|
||||
"Admin released PPQ auto top-up claim after manual reconciliation",
|
||||
extra={"provider_id": provider_id, "state_token": payload.state_token},
|
||||
)
|
||||
return {"ok": True, "released": True}
|
||||
|
||||
|
||||
@admin_router.get("/api/transactions", dependencies=[Depends(require_admin_api)])
|
||||
async def get_transactions_api(
|
||||
type: str | None = None,
|
||||
@@ -1633,7 +1815,11 @@ async def get_transactions_api(
|
||||
async with create_session() as session:
|
||||
from sqlmodel import col, func
|
||||
|
||||
base = select(CashuTransaction)
|
||||
# Hide only the deterministic PPQ claim-lock rows. Append-only PPQ
|
||||
# payment rows remain visible as the audit trail for irreversible melts.
|
||||
base = select(CashuTransaction).where(
|
||||
~col(CashuTransaction.id).like("ppq-auto-topup-%")
|
||||
)
|
||||
if type:
|
||||
base = base.where(CashuTransaction.type == type)
|
||||
if source:
|
||||
@@ -1669,12 +1855,18 @@ async def get_transactions_api(
|
||||
)
|
||||
total = count_result.one()
|
||||
|
||||
stmt = base.order_by(col(CashuTransaction.created_at).desc()).offset(offset).limit(limit)
|
||||
stmt = (
|
||||
base.order_by(col(CashuTransaction.created_at).desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
results = await session.exec(stmt)
|
||||
transactions = results.all()
|
||||
|
||||
return {
|
||||
"transactions": [tx.dict() for tx in transactions],
|
||||
"transactions": [
|
||||
tx.dict(exclude={"sweep_started_at"}) for tx in transactions
|
||||
],
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
+303
-33
@@ -12,21 +12,80 @@ from typing import AsyncGenerator
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from alembic.util.exc import CommandError
|
||||
from sqlalchemy import UniqueConstraint, delete
|
||||
from sqlalchemy import Index, UniqueConstraint, case, delete, event, or_
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.exc import IntegrityError, OperationalError
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlalchemy.ext.asyncio.engine import create_async_engine
|
||||
from sqlalchemy.orm import aliased
|
||||
from sqlmodel import Field, Relationship, SQLModel, col, func, select, update
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from .logging import get_logger
|
||||
from .settings import settings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///keys.db")
|
||||
|
||||
|
||||
engine = create_async_engine(DATABASE_URL, echo=False) # echo=True for debugging SQL
|
||||
def create_db_engine(database_url: str = DATABASE_URL) -> AsyncEngine:
|
||||
"""Build and instrument an async engine from environment-only settings."""
|
||||
url = make_url(database_url)
|
||||
backend = url.get_backend_name()
|
||||
is_sqlite = backend == "sqlite"
|
||||
is_memory_sqlite = is_sqlite and url.database in {None, "", ":memory:"}
|
||||
pool_pre_ping = settings.database_pool_pre_ping or not is_sqlite
|
||||
options: dict[str, int | float | bool] = {"pool_pre_ping": pool_pre_ping}
|
||||
if not is_memory_sqlite:
|
||||
options.update(
|
||||
pool_size=settings.database_pool_size,
|
||||
max_overflow=settings.database_max_overflow,
|
||||
pool_timeout=settings.database_pool_timeout,
|
||||
pool_recycle=settings.database_pool_recycle,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Database pool configured",
|
||||
extra={
|
||||
"database_url_backend": backend,
|
||||
"in_memory_sqlite": is_memory_sqlite,
|
||||
**options,
|
||||
},
|
||||
)
|
||||
created_engine = create_async_engine(database_url, echo=False, **options)
|
||||
hold_warn_seconds = settings.database_pool_hold_warn_seconds
|
||||
|
||||
def record_pool_checkout(
|
||||
dbapi_connection: object, connection_record: object, proxy: object
|
||||
) -> None:
|
||||
connection_record.info["routstr_checked_out_at"] = time.monotonic() # type: ignore[attr-defined]
|
||||
|
||||
def record_pool_checkin(
|
||||
dbapi_connection: object, connection_record: object
|
||||
) -> None:
|
||||
checked_out_at = connection_record.info.pop( # type: ignore[attr-defined]
|
||||
"routstr_checked_out_at", None
|
||||
)
|
||||
if checked_out_at is None:
|
||||
return
|
||||
held_seconds = time.monotonic() - checked_out_at
|
||||
if held_seconds >= hold_warn_seconds:
|
||||
logger.warning(
|
||||
"Database connection held longer than threshold",
|
||||
extra={
|
||||
"held_seconds": round(held_seconds, 3),
|
||||
"threshold_seconds": hold_warn_seconds,
|
||||
"pool_status": created_engine.pool.status(),
|
||||
},
|
||||
)
|
||||
|
||||
event.listen(created_engine.sync_engine, "checkout", record_pool_checkout)
|
||||
event.listen(created_engine.sync_engine, "checkin", record_pool_checkin)
|
||||
return created_engine
|
||||
|
||||
|
||||
engine = create_db_engine()
|
||||
|
||||
|
||||
class ApiKey(SQLModel, table=True): # type: ignore
|
||||
@@ -99,32 +158,133 @@ class ApiKey(SQLModel, table=True): # type: ignore
|
||||
|
||||
|
||||
async def reset_all_reserved_balances(session: AsyncSession) -> None:
|
||||
stmt = update(ApiKey).values(reserved_balance=0, reserved_at=None)
|
||||
await session.exec(stmt) # type: ignore[call-overload]
|
||||
"""Release every active durable reservation during explicit startup reset."""
|
||||
await session.exec( # type: ignore[call-overload]
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.values(status="released")
|
||||
)
|
||||
await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey).values(reserved_balance=0, reserved_at=None)
|
||||
)
|
||||
await session.commit()
|
||||
logger.info("Reset reserved balances on startup")
|
||||
|
||||
|
||||
async def release_stale_reservations(
|
||||
session: AsyncSession, max_age_seconds: int
|
||||
session: AsyncSession,
|
||||
max_age_seconds: int,
|
||||
*,
|
||||
key_hash: str | None = None,
|
||||
) -> int:
|
||||
"""Release reservations whose last reserve is older than max_age_seconds.
|
||||
"""
|
||||
"""Release stale durable reservations without touching newer reservations."""
|
||||
cutoff = int(time.time()) - max_age_seconds
|
||||
stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.reserved_balance) > 0)
|
||||
.where(col(ApiKey.reserved_at).is_not(None))
|
||||
.where(col(ApiKey.reserved_at) < cutoff)
|
||||
.values(reserved_balance=0, reserved_at=None)
|
||||
query = (
|
||||
select(ReservationRelease)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.created_at) < cutoff)
|
||||
)
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
if key_hash is not None:
|
||||
query = query.where(
|
||||
or_(
|
||||
col(ReservationRelease.key_hash) == key_hash,
|
||||
col(ReservationRelease.billing_key_hash) == key_hash,
|
||||
)
|
||||
)
|
||||
reservations = (await session.exec(query)).all()
|
||||
released = 0
|
||||
|
||||
for reservation in reservations:
|
||||
transition = await session.exec( # type: ignore[call-overload]
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == reservation.id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.values(status="released")
|
||||
)
|
||||
if transition.rowcount != 1:
|
||||
continue
|
||||
|
||||
values = {
|
||||
"reserved_balance": col(ApiKey.reserved_balance)
|
||||
- reservation.reserved_msats,
|
||||
"reserved_at": case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) - reservation.reserved_msats > 0,
|
||||
col(ApiKey.reserved_at),
|
||||
),
|
||||
else_=None,
|
||||
),
|
||||
}
|
||||
parent_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == reservation.billing_key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
if parent_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return 0
|
||||
|
||||
if reservation.billing_key_hash != reservation.key_hash:
|
||||
child_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == reservation.key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
if child_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return 0
|
||||
released += 1
|
||||
|
||||
# Rolling upgrades can leave aggregate reservations created before durable
|
||||
# reservation rows existed. Release only stale aggregates that have no active
|
||||
# durable owner; targeted refund cleanup also heals legacy NULL timestamps.
|
||||
legacy_query = select(ApiKey).where(col(ApiKey.reserved_balance) > 0)
|
||||
if key_hash is None:
|
||||
legacy_query = legacy_query.where(col(ApiKey.reserved_at).is_not(None)).where(
|
||||
col(ApiKey.reserved_at) < cutoff
|
||||
)
|
||||
else:
|
||||
legacy_query = legacy_query.where(
|
||||
or_(
|
||||
col(ApiKey.hashed_key) == key_hash,
|
||||
col(ApiKey.parent_key_hash) == key_hash,
|
||||
)
|
||||
).where(
|
||||
or_(col(ApiKey.reserved_at).is_(None), col(ApiKey.reserved_at) < cutoff)
|
||||
)
|
||||
|
||||
for legacy_key in (await session.exec(legacy_query)).all():
|
||||
active_owner = (
|
||||
await session.exec(
|
||||
select(ReservationRelease.id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(
|
||||
or_(
|
||||
col(ReservationRelease.key_hash) == legacy_key.hashed_key,
|
||||
col(ReservationRelease.billing_key_hash)
|
||||
== legacy_key.hashed_key,
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).first()
|
||||
if active_owner is not None:
|
||||
continue
|
||||
legacy_key.reserved_balance = 0
|
||||
legacy_key.reserved_at = None
|
||||
session.add(legacy_key)
|
||||
released += 1
|
||||
|
||||
await session.commit()
|
||||
released = int(result.rowcount or 0)
|
||||
if released:
|
||||
logger.warning(
|
||||
"Released stale balance reservations",
|
||||
extra={"released_keys": released, "max_age_seconds": max_age_seconds},
|
||||
"Released stale reservations",
|
||||
extra={
|
||||
"released_reservations": released,
|
||||
"max_age_seconds": max_age_seconds,
|
||||
},
|
||||
)
|
||||
return released
|
||||
|
||||
@@ -133,7 +293,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
"""Delete dead parentless API keys; return the count removed.
|
||||
|
||||
Dead = 0 balance/reservation/spend/requests, older than the grace period,
|
||||
no parent, no children, no pending invoice. Cashu rows are unlinked (not
|
||||
no parent, no children, no retryable invoice. Cashu rows are unlinked (not
|
||||
deleted) first to keep the audit trail.
|
||||
"""
|
||||
cutoff = int(time.time()) - min_age_seconds
|
||||
@@ -147,7 +307,9 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
pending_invoice = (
|
||||
select(LightningInvoice.id)
|
||||
.where(col(LightningInvoice.api_key_hash) == col(ApiKey.hashed_key))
|
||||
.where(col(LightningInvoice.status) == "pending")
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(("pending", "settlement_pending"))
|
||||
)
|
||||
).exists()
|
||||
|
||||
eligible_hashes = (
|
||||
@@ -157,9 +319,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
.where(col(ApiKey.total_spent) == 0)
|
||||
.where(col(ApiKey.total_requests) == 0)
|
||||
.where(col(ApiKey.parent_key_hash).is_(None))
|
||||
.where(
|
||||
(col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff)
|
||||
)
|
||||
.where((col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff))
|
||||
.where(~pending_invoice)
|
||||
.where(~has_children)
|
||||
)
|
||||
@@ -213,6 +373,60 @@ class ModelRow(SQLModel, table=True): # type: ignore
|
||||
upstream_provider: "UpstreamProviderRow" = Relationship(back_populates="models")
|
||||
|
||||
|
||||
class ModelPathRow(SQLModel, table=True): # type: ignore
|
||||
"""Upstream provider path a model is reachable through.
|
||||
|
||||
Discovery/visibility data only. ``model_id`` is intentionally NOT globally
|
||||
unique: it is the client-visible ``/v1/models`` id (``forwarded_model_id or
|
||||
id``) grouped across every provider that exposes the model. A single model
|
||||
can therefore have several rows — one per direct provider path plus one per
|
||||
OpenRouter sub-provider endpoint.
|
||||
"""
|
||||
|
||||
__tablename__ = "model_paths"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"model_id",
|
||||
"path",
|
||||
"upstream_provider_id",
|
||||
name="uq_model_paths_model_path_provider",
|
||||
),
|
||||
)
|
||||
id: int | None = Field(default=None, primary_key=True)
|
||||
# No standalone index on model_id: the unique constraint's autoindex already
|
||||
# leads on model_id, so a second index only adds write amplification.
|
||||
model_id: str = Field(
|
||||
description="Client-visible /v1/models id (forwarded_model_id or id)"
|
||||
)
|
||||
path: str = Field(
|
||||
description=(
|
||||
"Opaque selector containing upstream URL, provider ID, model ID, "
|
||||
"and optional endpoint tag"
|
||||
)
|
||||
)
|
||||
provider_slug: str = Field(
|
||||
description="Public slug of the configured upstream provider"
|
||||
)
|
||||
provider_type: str = Field(description="Configured upstream provider type")
|
||||
endpoint_tag: str | None = Field(
|
||||
default=None,
|
||||
description="Exact OpenRouter endpoint tag used for request-side selection",
|
||||
)
|
||||
endpoint_name: str | None = Field(
|
||||
default=None, description="Human-readable endpoint display name"
|
||||
)
|
||||
upstream_provider_id: int = Field(
|
||||
index=True,
|
||||
foreign_key="upstream_providers.id",
|
||||
ondelete="CASCADE",
|
||||
description="upstream_providers.id this path was discovered from",
|
||||
)
|
||||
updated_at: int = Field(
|
||||
default=0,
|
||||
description="Unix timestamp of the refresh cycle that wrote this row",
|
||||
)
|
||||
|
||||
|
||||
class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "lightning_invoices"
|
||||
|
||||
@@ -222,12 +436,19 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
description: str = Field(description="Invoice description")
|
||||
payment_hash: str = Field(description="Payment hash for tracking", unique=True)
|
||||
status: str = Field(
|
||||
default="pending", description="pending, paid, expired, cancelled"
|
||||
default="pending",
|
||||
description=(
|
||||
"pending, settlement_pending, paid, expired, cancelled, "
|
||||
"reconciliation_required"
|
||||
),
|
||||
)
|
||||
api_key_hash: str | None = Field(
|
||||
default=None, description="Associated API key hash for topup operations"
|
||||
)
|
||||
purpose: str = Field(description="create or topup")
|
||||
mint_url: str | None = Field(
|
||||
default=None, description="Mint URL where the quote was created (fallback tracking)"
|
||||
)
|
||||
created_at: int = Field(
|
||||
default_factory=lambda: int(time.time()), description="Unix timestamp"
|
||||
)
|
||||
@@ -267,6 +488,10 @@ class CashuTransaction(SQLModel, table=True): # type: ignore
|
||||
)
|
||||
collected: bool = Field(default=False)
|
||||
swept: bool = Field(default=False)
|
||||
sweep_started_at: int | None = Field(
|
||||
default=None,
|
||||
description="Unix timestamp for a recoverable refund-sweep claim",
|
||||
)
|
||||
source: str = Field(
|
||||
default="x-cashu",
|
||||
description="Payment source: x-cashu or apikey",
|
||||
@@ -434,6 +659,20 @@ class UpstreamProviderRow(SQLModel, table=True): # type: ignore
|
||||
)
|
||||
|
||||
|
||||
class ReservationRelease(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "reservation_releases"
|
||||
__table_args__ = (
|
||||
Index("ix_reservation_releases_status_created_at", "status", "created_at"),
|
||||
)
|
||||
|
||||
id: str = Field(primary_key=True)
|
||||
key_hash: str = Field(index=True)
|
||||
billing_key_hash: str = Field(index=True)
|
||||
reserved_msats: int
|
||||
status: str = Field(default="active")
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
|
||||
|
||||
class RoutstrFee(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "routstr_fees"
|
||||
id: int = Field(default=1, primary_key=True)
|
||||
@@ -484,9 +723,7 @@ class CliToken(SQLModel, table=True): # type: ignore
|
||||
"""Long-lived authorization token for CLI/agent use against admin endpoints."""
|
||||
|
||||
__tablename__ = "cli_tokens"
|
||||
id: str = Field(
|
||||
primary_key=True, default_factory=lambda: uuid.uuid4().hex
|
||||
)
|
||||
id: str = Field(primary_key=True, default_factory=lambda: uuid.uuid4().hex)
|
||||
token: str = Field(unique=True, index=True, description="Bearer token value")
|
||||
name: str = Field(description="Human-readable label for this token")
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
@@ -585,9 +822,7 @@ async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> bool:
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def complete_routstr_fee_payout(
|
||||
session: AsyncSession, paid_msats: int
|
||||
) -> bool:
|
||||
async def complete_routstr_fee_payout(session: AsyncSession, paid_msats: int) -> bool:
|
||||
"""Mark a checkpointed payout complete after the external payment succeeds."""
|
||||
stmt = (
|
||||
update(RoutstrFee)
|
||||
@@ -605,14 +840,49 @@ async def complete_routstr_fee_payout(
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def balances_for_mint_and_unit(
|
||||
async def total_user_liability(db_session: AsyncSession) -> int:
|
||||
"""Return all outstanding API-key balances in millisatoshis."""
|
||||
result = await db_session.exec(select(func.sum(ApiKey.balance)))
|
||||
return int(result.one() or 0)
|
||||
|
||||
|
||||
async def balance_for_mint_and_unit(
|
||||
db_session: AsyncSession, mint_url: str, unit: str
|
||||
) -> int:
|
||||
query = select(func.sum(ApiKey.balance)).where(
|
||||
ApiKey.refund_mint_url == mint_url, ApiKey.refund_currency == unit
|
||||
"""Return the user liability for one mint and unit in millisatoshis."""
|
||||
result = await db_session.exec(
|
||||
select(func.sum(ApiKey.balance)).where(
|
||||
col(ApiKey.refund_mint_url) == mint_url,
|
||||
col(ApiKey.refund_currency) == unit,
|
||||
)
|
||||
)
|
||||
return int(result.one() or 0)
|
||||
|
||||
|
||||
async def balances_by_mint_and_unit(
|
||||
db_session: AsyncSession, mint_urls: list[str], units: list[str]
|
||||
) -> dict[tuple[str, str], int]:
|
||||
"""Return requested user liabilities grouped by mint and unit."""
|
||||
if not mint_urls or not units:
|
||||
return {}
|
||||
query = (
|
||||
select(
|
||||
col(ApiKey.refund_mint_url),
|
||||
col(ApiKey.refund_currency),
|
||||
func.sum(ApiKey.balance),
|
||||
)
|
||||
.where(
|
||||
col(ApiKey.refund_mint_url).in_(mint_urls),
|
||||
col(ApiKey.refund_currency).in_(units),
|
||||
)
|
||||
.group_by(col(ApiKey.refund_mint_url), col(ApiKey.refund_currency))
|
||||
)
|
||||
result = await db_session.exec(query)
|
||||
return result.one() or 0
|
||||
return {
|
||||
(mint_url, unit): int(balance or 0)
|
||||
for mint_url, unit, balance in result.all()
|
||||
if mint_url is not None and unit is not None
|
||||
}
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
|
||||
+15
-9
@@ -58,6 +58,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
providers_task = None
|
||||
models_refresh_task = None
|
||||
model_maps_refresh_task = None
|
||||
model_paths_refresh_task = None
|
||||
key_reset_task = None
|
||||
stale_reservation_task = None
|
||||
dead_key_prune_task = None
|
||||
@@ -130,6 +131,13 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
refresh_upstreams_models_periodically(get_upstreams)
|
||||
)
|
||||
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
|
||||
# Always started: the loop re-reads the enable flag and interval every
|
||||
# iteration, so 0 -> N (or re-enabling) takes effect without a restart.
|
||||
from ..upstream.model_paths import refresh_model_paths_periodically
|
||||
|
||||
model_paths_refresh_task = asyncio.create_task(
|
||||
refresh_model_paths_periodically(get_upstreams)
|
||||
)
|
||||
payout_task = asyncio.create_task(periodic_payout())
|
||||
if global_settings.nsec:
|
||||
nip91_task = asyncio.create_task(announce_provider())
|
||||
@@ -137,9 +145,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
if global_settings.providers_refresh_interval_seconds > 0:
|
||||
providers_task = asyncio.create_task(providers_cache_refresher())
|
||||
key_reset_task = asyncio.create_task(periodic_key_reset())
|
||||
stale_reservation_task = asyncio.create_task(
|
||||
periodic_stale_reservation_sweep()
|
||||
)
|
||||
stale_reservation_task = asyncio.create_task(periodic_stale_reservation_sweep())
|
||||
dead_key_prune_task = asyncio.create_task(periodic_dead_key_prune())
|
||||
auto_topup_task = asyncio.create_task(periodic_auto_topup())
|
||||
refund_sweep_task = asyncio.create_task(periodic_refund_sweep())
|
||||
@@ -176,6 +182,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
models_refresh_task.cancel()
|
||||
if model_maps_refresh_task is not None:
|
||||
model_maps_refresh_task.cancel()
|
||||
if model_paths_refresh_task is not None:
|
||||
model_paths_refresh_task.cancel()
|
||||
if key_reset_task is not None:
|
||||
key_reset_task.cancel()
|
||||
if stale_reservation_task is not None:
|
||||
@@ -209,6 +217,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
tasks_to_wait.append(models_refresh_task)
|
||||
if model_maps_refresh_task is not None:
|
||||
tasks_to_wait.append(model_maps_refresh_task)
|
||||
if model_paths_refresh_task is not None:
|
||||
tasks_to_wait.append(model_paths_refresh_task)
|
||||
if key_reset_task is not None:
|
||||
tasks_to_wait.append(key_reset_task)
|
||||
if stale_reservation_task is not None:
|
||||
@@ -245,9 +255,7 @@ class _ImmutableStaticFiles(StaticFiles):
|
||||
async def get_response(self, path: str, scope: Scope) -> StarletteResponse:
|
||||
response = await super().get_response(path, scope)
|
||||
if response.status_code == 200:
|
||||
response.headers["Cache-Control"] = (
|
||||
"public, max-age=31536000, immutable"
|
||||
)
|
||||
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
return response
|
||||
|
||||
|
||||
@@ -321,9 +329,7 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir():
|
||||
# Serve the App Router RSC payload for the home page.
|
||||
@app.get("/index.txt", include_in_schema=False)
|
||||
async def serve_root_rsc() -> FileResponse:
|
||||
return FileResponse(
|
||||
UI_DIST_PATH / "index.txt", media_type="text/x-component"
|
||||
)
|
||||
return FileResponse(UI_DIST_PATH / "index.txt", media_type="text/x-component")
|
||||
|
||||
# Next.js is built with `trailingSlash: true`, so all UI page URLs end
|
||||
# with a slash (e.g. `/login/`). The proxy router catches `/{path:path}`
|
||||
|
||||
@@ -41,6 +41,9 @@ class Settings(BaseSettings):
|
||||
receive_ln_address: str = Field(default="", env="RECEIVE_LN_ADDRESS")
|
||||
primary_mint: str = Field(default="", env="PRIMARY_MINT_URL")
|
||||
primary_mint_unit: str = Field(default="sat", env="PRIMARY_MINT_UNIT")
|
||||
mint_operation_concurrency: int = Field(
|
||||
default=4, ge=1, env="MINT_OPERATION_CONCURRENCY"
|
||||
)
|
||||
|
||||
# Lightning payout configuration
|
||||
# Minimum available balance (in satoshis) before profit is paid out over
|
||||
@@ -50,6 +53,18 @@ class Settings(BaseSettings):
|
||||
payout_interval_seconds: int = Field(
|
||||
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
|
||||
)
|
||||
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
|
||||
# checkstate). When a mint is slow or rate-limiting, operations are
|
||||
# cancelled after this delay instead of hanging indefinitely.
|
||||
mint_operation_timeout_seconds: int = Field(
|
||||
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
|
||||
)
|
||||
# Maximum concurrent API operations per mint. Actual mint quotas vary by
|
||||
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
|
||||
# pacing. 0 = unlimited concurrency.
|
||||
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
|
||||
# Max retries when a mint returns 429 or times out (exponential backoff).
|
||||
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
|
||||
|
||||
# Pricing
|
||||
# Default behavior: derive pricing from MODELS
|
||||
@@ -95,10 +110,36 @@ class Settings(BaseSettings):
|
||||
models_refresh_interval_seconds: int = Field(
|
||||
default=360, env="MODELS_REFRESH_INTERVAL_SECONDS"
|
||||
)
|
||||
model_paths_refresh_interval_seconds: int = Field(
|
||||
default=600, env="MODEL_PATHS_REFRESH_INTERVAL_SECONDS"
|
||||
)
|
||||
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
|
||||
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
|
||||
enable_model_paths_refresh: bool = Field(
|
||||
default=True, env="ENABLE_MODEL_PATHS_REFRESH"
|
||||
)
|
||||
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(default=604800, env="REFUND_SWEEP_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(
|
||||
default=604800, env="REFUND_SWEEP_TTL_SECONDS"
|
||||
)
|
||||
refund_sweep_claim_timeout_seconds: int = Field(
|
||||
default=900, gt=0, env="REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS"
|
||||
)
|
||||
|
||||
# Database connection-pool controls (advanced). Capacity defaults provide
|
||||
# headroom for Routstr's concurrent request and background-payment workload.
|
||||
# Pre-ping is enabled by the engine factory for networked backends; SQLite
|
||||
# can explicitly opt in. These fields are env-only below.
|
||||
database_pool_size: int = Field(default=10, ge=1, env="DATABASE_POOL_SIZE")
|
||||
database_max_overflow: int = Field(default=20, ge=0, env="DATABASE_MAX_OVERFLOW")
|
||||
database_pool_timeout: float = Field(
|
||||
default=15.0, gt=0, env="DATABASE_POOL_TIMEOUT"
|
||||
)
|
||||
database_pool_recycle: int = Field(default=1800, ge=0, env="DATABASE_POOL_RECYCLE")
|
||||
database_pool_pre_ping: bool = Field(default=False, env="DATABASE_POOL_PRE_PING")
|
||||
database_pool_hold_warn_seconds: float = Field(
|
||||
default=10.0, gt=0, env="DATABASE_POOL_HOLD_WARN_SECONDS"
|
||||
)
|
||||
|
||||
# Logging
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
@@ -117,9 +158,8 @@ class Settings(BaseSettings):
|
||||
|
||||
# Discovery
|
||||
relays: list[str] = Field(default_factory=list, env="RELAYS")
|
||||
enable_analytics_sharing: bool = Field(
|
||||
default=True, env="ENABLE_ANALYTICS_SHARING"
|
||||
)
|
||||
enable_analytics_sharing: bool = Field(default=True, env="ENABLE_ANALYTICS_SHARING")
|
||||
|
||||
|
||||
def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Discard unknown keys from persisted settings."""
|
||||
@@ -144,10 +184,32 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
|
||||
# ``routstr.core.vault``.
|
||||
SECRET_FIELDS = frozenset({"admin_password", "nsec"})
|
||||
|
||||
# Infrastructure the node needs *before* it can open a DB session — so it can
|
||||
# never be configured from the DB (chicken-and-egg) and stays env-only. Unlike
|
||||
# secrets (owned by bootstrap), these are excluded so the DB settings blob can
|
||||
# neither store nor shadow them; env is always authoritative.
|
||||
ENV_ONLY_FIELDS = frozenset(
|
||||
{
|
||||
"database_pool_size",
|
||||
"database_max_overflow",
|
||||
"database_pool_timeout",
|
||||
"database_pool_recycle",
|
||||
"database_pool_pre_ping",
|
||||
"database_pool_hold_warn_seconds",
|
||||
}
|
||||
)
|
||||
|
||||
_NON_PERSISTED_FIELDS = SECRET_FIELDS | ENV_ONLY_FIELDS
|
||||
|
||||
|
||||
def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return a copy of ``data`` without any secret fields (for persistence)."""
|
||||
return {k: v for k, v in data.items() if k not in SECRET_FIELDS}
|
||||
"""Return a copy of ``data`` without secret or env-only fields.
|
||||
|
||||
Both are kept out of the persisted settings blob: secrets for confidentiality,
|
||||
env-only fields (e.g. DB pool sizing) because they must never be sourced from
|
||||
the database.
|
||||
"""
|
||||
return {k: v for k, v in data.items() if k not in _NON_PERSISTED_FIELDS}
|
||||
|
||||
|
||||
def _apply_to_live_settings(data: dict[str, Any]) -> None:
|
||||
@@ -327,7 +389,13 @@ class SettingsService:
|
||||
valid_fields = set(env_resolved.dict().keys())
|
||||
merged_dict: dict[str, Any] = dict(env_resolved.dict())
|
||||
merged_dict.update(
|
||||
{k: v for k, v in db_json.items() if v not in (None, "", [], {}) and k in valid_fields}
|
||||
{
|
||||
k: v
|
||||
for k, v in db_json.items()
|
||||
if v not in (None, "", [], {})
|
||||
and k in valid_fields
|
||||
and k not in ENV_ONLY_FIELDS
|
||||
}
|
||||
)
|
||||
merged_dict = Settings(**merged_dict).dict()
|
||||
|
||||
@@ -394,8 +462,13 @@ class SettingsService:
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
# Update in-place
|
||||
# Update in-place. Env-only fields (e.g. DB pool sizing) are never
|
||||
# applied here: the engine pool is already built at boot from env,
|
||||
# so letting an update mutate the live value would only make it
|
||||
# diverge from the running pool.
|
||||
for k, v in candidate.dict().items():
|
||||
if k in ENV_ONLY_FIELDS:
|
||||
continue
|
||||
setattr(settings, k, v)
|
||||
cls._current = settings
|
||||
return settings
|
||||
|
||||
+571
-86
@@ -1,22 +1,101 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncGenerator
|
||||
|
||||
from cashu.core.base import MintQuoteState
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlmodel import col, select
|
||||
from sqlalchemy.orm.attributes import set_committed_value
|
||||
from sqlmodel import col, select, update
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from .core.db import ApiKey, LightningInvoice, create_session, get_session
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .wallet import get_wallet
|
||||
from .mint import (
|
||||
is_mint_rate_limited,
|
||||
mint_cooldown_remaining,
|
||||
run_mint_operation,
|
||||
)
|
||||
from .wallet import (
|
||||
MintConnectionError,
|
||||
get_wallet,
|
||||
is_mint_connection_error,
|
||||
wallet_operation_guard,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
lightning_router = APIRouter(prefix="/lightning")
|
||||
|
||||
# Avoid duplicate work within one process. Cross-process settlement is fenced
|
||||
# by claiming a paid quote before minting and by the final conditional update.
|
||||
@dataclass
|
||||
class _InvoiceLockEntry:
|
||||
lock: asyncio.Lock
|
||||
users: int = 0
|
||||
|
||||
|
||||
_invoice_settlement_locks: dict[str, _InvoiceLockEntry] = {}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _invoice_settlement_lock(invoice_id: str) -> AsyncGenerator[None, None]:
|
||||
"""Serialize one invoice and remove its lock after the last waiter leaves."""
|
||||
|
||||
entry = _invoice_settlement_locks.get(invoice_id)
|
||||
if entry is None:
|
||||
entry = _InvoiceLockEntry(asyncio.Lock())
|
||||
_invoice_settlement_locks[invoice_id] = entry
|
||||
entry.users += 1
|
||||
try:
|
||||
async with entry.lock:
|
||||
yield
|
||||
finally:
|
||||
entry.users -= 1
|
||||
if entry.users == 0 and _invoice_settlement_locks.get(invoice_id) is entry:
|
||||
del _invoice_settlement_locks[invoice_id]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _InvoiceSettlement:
|
||||
id: str
|
||||
payment_hash: str
|
||||
amount_sats: int
|
||||
purpose: str
|
||||
api_key_hash: str | None
|
||||
mint_url: str | None
|
||||
balance_limit: int | None
|
||||
balance_limit_reset: str | None
|
||||
validity_date: int | None
|
||||
|
||||
@classmethod
|
||||
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
|
||||
return cls(
|
||||
id=invoice.id,
|
||||
payment_hash=invoice.payment_hash,
|
||||
amount_sats=invoice.amount_sats,
|
||||
purpose=invoice.purpose,
|
||||
api_key_hash=invoice.api_key_hash,
|
||||
mint_url=invoice.mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
)
|
||||
|
||||
|
||||
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
|
||||
"""Update a caller view without marking a mapped object dirty."""
|
||||
try:
|
||||
set_committed_value(invoice, key, value)
|
||||
except AttributeError:
|
||||
setattr(invoice, key, value)
|
||||
|
||||
|
||||
class InvoiceCreateRequest(BaseModel):
|
||||
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
|
||||
@@ -60,16 +139,102 @@ class InvoiceStatusResponse(BaseModel):
|
||||
expires_at: int
|
||||
|
||||
|
||||
_RETRYABLE_INVOICE_STATUSES = ("pending", "settlement_pending")
|
||||
|
||||
|
||||
class InvoiceRecoverRequest(BaseModel):
|
||||
bolt11: str = Field(description="BOLT11 invoice string")
|
||||
|
||||
|
||||
def _trusted_mint_candidates() -> list[str]:
|
||||
return [
|
||||
mint
|
||||
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
|
||||
if mint
|
||||
]
|
||||
|
||||
|
||||
async def _request_mint_with_fallback(
|
||||
amount_sats: int,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
"""Request a quote, falling back only among the allowed trusted mints.
|
||||
|
||||
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
|
||||
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
|
||||
cryptic validation error deep in the stack. Fail fast with context.
|
||||
"""
|
||||
if amount_sats <= 0:
|
||||
raise ValueError(
|
||||
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
|
||||
)
|
||||
tried: list[str] = []
|
||||
trusted = _trusted_mint_candidates()
|
||||
if allowed_mints:
|
||||
# Persisted mint preferences (e.g. an API key's refund_mint_url) must
|
||||
# not outlive the operator's trusted-mint configuration.
|
||||
candidates = [m for m in dict.fromkeys(allowed_mints) if m in trusted]
|
||||
if not candidates:
|
||||
logger.warning(
|
||||
"Requested mints are no longer trusted; falling back to "
|
||||
"configured mints",
|
||||
extra={
|
||||
"requested_mints": list(dict.fromkeys(allowed_mints)),
|
||||
"op_name": "request_mint_invoice",
|
||||
},
|
||||
)
|
||||
candidates = trusted
|
||||
else:
|
||||
candidates = trusted
|
||||
for mint_url in candidates:
|
||||
cooldown = mint_cooldown_remaining(mint_url)
|
||||
if cooldown > 0:
|
||||
tried.append(f"{mint_url}: cooling down")
|
||||
logger.info(
|
||||
"Skipping mint during cooldown",
|
||||
extra={
|
||||
"mint_url": mint_url,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"op_name": "request_mint_invoice",
|
||||
},
|
||||
)
|
||||
continue
|
||||
try:
|
||||
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
|
||||
quote = await run_mint_operation(
|
||||
lambda: wallet.request_mint(amount_sats),
|
||||
op_name="request_mint_invoice",
|
||||
mint_url=mint_url,
|
||||
retry_on_rate_limit=False,
|
||||
)
|
||||
return quote.request, quote.quote, mint_url
|
||||
except Exception as e:
|
||||
tried.append(f"{mint_url}: {type(e).__name__}")
|
||||
if not is_mint_connection_error(e) and not is_mint_rate_limited(e):
|
||||
raise
|
||||
logger.warning(
|
||||
"request_mint failed, trying fallback mint",
|
||||
extra={
|
||||
"failed_mint": mint_url,
|
||||
"error": str(e),
|
||||
"tried": tried,
|
||||
},
|
||||
)
|
||||
continue
|
||||
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
|
||||
|
||||
|
||||
async def generate_lightning_invoice(
|
||||
amount_sats: int, description: str
|
||||
) -> tuple[str, str]:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
quote = await wallet.request_mint(amount_sats)
|
||||
return quote.request, quote.quote
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
|
||||
amount_sats, allowed_mints=allowed_mints
|
||||
)
|
||||
return bolt11, payment_hash, mint_url
|
||||
|
||||
|
||||
def generate_invoice_id() -> str:
|
||||
@@ -83,6 +248,7 @@ async def create_invoice(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> InvoiceCreateResponse:
|
||||
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
|
||||
topup_api_key: ApiKey | None = None
|
||||
|
||||
if request.purpose == "topup":
|
||||
if not api_key_token:
|
||||
@@ -93,14 +259,23 @@ async def create_invoice(
|
||||
if not api_key_token.startswith("sk-"):
|
||||
raise HTTPException(status_code=400, detail="Invalid API key format")
|
||||
|
||||
api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not api_key:
|
||||
topup_api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not topup_api_key:
|
||||
raise HTTPException(status_code=404, detail="API key not found")
|
||||
|
||||
try:
|
||||
description = f"Routstr {request.purpose} {request.amount_sats} sats"
|
||||
bolt11, payment_hash = await generate_lightning_invoice(
|
||||
request.amount_sats, description
|
||||
allowed_mints = None
|
||||
if request.purpose == "topup":
|
||||
assert topup_api_key is not None
|
||||
# A key's liabilities are attributed to a single refund mint. Keep
|
||||
# top-up collateral on that same mint so balances and payouts cannot
|
||||
# misclassify funds held by another mint as owner profit.
|
||||
allowed_mints = [
|
||||
topup_api_key.refund_mint_url or settings.primary_mint
|
||||
]
|
||||
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
|
||||
request.amount_sats, description, allowed_mints=allowed_mints
|
||||
)
|
||||
|
||||
invoice_id = generate_invoice_id()
|
||||
@@ -115,6 +290,7 @@ async def create_invoice(
|
||||
status="pending",
|
||||
api_key_hash=api_key_token[3:] if api_key_token else None,
|
||||
purpose=request.purpose,
|
||||
mint_url=mint_url,
|
||||
balance_limit=request.balance_limit,
|
||||
balance_limit_reset=request.balance_limit_reset,
|
||||
validity_date=request.validity_date,
|
||||
@@ -160,12 +336,12 @@ async def get_invoice_status(
|
||||
if not invoice:
|
||||
raise HTTPException(status_code=404, detail="Invoice not found")
|
||||
|
||||
if invoice.status == "pending":
|
||||
await check_invoice_payment(invoice, session)
|
||||
|
||||
if invoice.status == "pending" and int(time.time()) > invoice.expires_at:
|
||||
invoice.status = "expired"
|
||||
await session.commit()
|
||||
definitively_unpaid = False
|
||||
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
|
||||
api_key = None
|
||||
if invoice.status == "paid" and invoice.purpose == "create":
|
||||
@@ -199,8 +375,12 @@ async def recover_invoice(
|
||||
if not invoice:
|
||||
raise HTTPException(status_code=404, detail="Invoice not found")
|
||||
|
||||
if invoice.status == "pending":
|
||||
await check_invoice_payment(invoice, session)
|
||||
definitively_unpaid = False
|
||||
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
|
||||
api_key = None
|
||||
if invoice.status == "paid":
|
||||
@@ -219,114 +399,419 @@ async def recover_invoice(
|
||||
)
|
||||
|
||||
|
||||
async def _claim_paid_invoice_for_settlement(
|
||||
invoice: LightningInvoice,
|
||||
caller_session: AsyncSession,
|
||||
observed_status: str,
|
||||
) -> bool:
|
||||
"""Claim an authoritative paid quote before consuming it at the mint."""
|
||||
if observed_status == "settlement_pending":
|
||||
return True
|
||||
if observed_status != "pending":
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
async with create_session() as claim_session:
|
||||
claim = await claim_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="settlement_pending")
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
await claim_session.commit()
|
||||
|
||||
if claim.rowcount != 1:
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
_publish_invoice_value(invoice, "status", "settlement_pending")
|
||||
return True
|
||||
|
||||
|
||||
async def check_invoice_payment(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
try:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
|
||||
mint_status = await wallet.get_mint_quote(invoice.payment_hash)
|
||||
|
||||
if mint_status.paid:
|
||||
invoice.status = "paid"
|
||||
invoice.paid_at = int(time.time())
|
||||
|
||||
if invoice.purpose == "create":
|
||||
api_key = await create_api_key_from_invoice(invoice, session)
|
||||
invoice.api_key_hash = api_key.hashed_key
|
||||
elif invoice.purpose == "topup" and invoice.api_key_hash:
|
||||
await topup_api_key_from_invoice(invoice, session)
|
||||
) -> bool:
|
||||
"""Settle an invoice and report whether its quote is definitively unpaid.
|
||||
|
||||
False covers paid, pending, and ambiguous transport/DB outcomes so callers
|
||||
never expire a quote merely because reconciliation could not complete.
|
||||
"""
|
||||
async with _invoice_settlement_lock(invoice.id), wallet_operation_guard():
|
||||
minted = False
|
||||
payment_confirmed = False
|
||||
try:
|
||||
# Snapshot the row and end the caller's read transaction before any
|
||||
# potentially slow mint I/O. All final DB mutations use owned,
|
||||
# short-lived sessions below.
|
||||
await session.refresh(invoice)
|
||||
if invoice.status not in _RETRYABLE_INVOICE_STATUSES:
|
||||
await session.commit()
|
||||
return False
|
||||
observed_status = invoice.status
|
||||
settlement = _InvoiceSettlement.from_invoice(invoice)
|
||||
await session.commit()
|
||||
|
||||
mint_url = settlement.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
try:
|
||||
mint_status = await run_mint_operation(
|
||||
lambda: wallet.get_mint_quote(settlement.payment_hash),
|
||||
op_name="get_mint_quote",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
except Exception as error:
|
||||
if not _is_quote_not_found(error):
|
||||
raise
|
||||
logger.info(
|
||||
"Invoice quote no longer exists at mint, marking expired",
|
||||
extra={"invoice_id": invoice.id, "error": str(error)},
|
||||
)
|
||||
return True
|
||||
if not mint_status.paid:
|
||||
return getattr(mint_status, "state", None) == MintQuoteState.unpaid
|
||||
payment_confirmed = True
|
||||
|
||||
# Fence expiry and other workers before consuming the paid quote.
|
||||
# If a concurrent expiry/finalization won, this worker must not mint.
|
||||
if not await _claim_paid_invoice_for_settlement(
|
||||
invoice, session, observed_status
|
||||
):
|
||||
return False
|
||||
|
||||
# Reject a paid top-up whose target was pruned before redeeming its
|
||||
# single-use quote. The validation session is closed before mint I/O.
|
||||
if settlement.purpose == "topup":
|
||||
if not settlement.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
async with create_session() as validation_session:
|
||||
target = await validation_session.get(
|
||||
ApiKey, settlement.api_key_hash
|
||||
)
|
||||
if target is None:
|
||||
terminal = await validation_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == settlement.id,
|
||||
col(LightningInvoice.status).in_(
|
||||
_RETRYABLE_INVOICE_STATUSES
|
||||
),
|
||||
)
|
||||
.values(status="reconciliation_required")
|
||||
)
|
||||
await validation_session.commit()
|
||||
if terminal.rowcount == 1:
|
||||
_publish_invoice_value(
|
||||
invoice, "status", "reconciliation_required"
|
||||
)
|
||||
else:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
logger.critical(
|
||||
"Paid topup invoice target API key was not found; reconciliation required",
|
||||
extra={"invoice_id": settlement.id},
|
||||
)
|
||||
return False
|
||||
|
||||
# Quote-linked proof verification makes an ambiguous mint response
|
||||
# retryable without crediting unrelated wallet balance growth.
|
||||
await _mint_invoice_quote(wallet, settlement)
|
||||
minted = True
|
||||
|
||||
paid_at = int(time.time())
|
||||
async with create_session() as finalization_session:
|
||||
settled, api_key_hash = await _finalize_invoice_settlement(
|
||||
settlement, finalization_session, paid_at
|
||||
)
|
||||
if not settled:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
return False
|
||||
|
||||
_publish_invoice_value(invoice, "status", "paid")
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
logger.info(
|
||||
"Lightning invoice paid",
|
||||
extra={
|
||||
"invoice_id": invoice.id,
|
||||
"amount_sats": invoice.amount_sats,
|
||||
"purpose": invoice.purpose,
|
||||
"api_key_hash": invoice.api_key_hash[:8] + "..."
|
||||
if invoice.api_key_hash
|
||||
"invoice_id": settlement.id,
|
||||
"amount_sats": settlement.amount_sats,
|
||||
"purpose": settlement.purpose,
|
||||
"api_key_hash": api_key_hash[:8] + "..."
|
||||
if api_key_hash
|
||||
else None,
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check invoice payment: {e}")
|
||||
return False
|
||||
except BaseException as error:
|
||||
# Never roll back the caller-owned session: doing so expires invoice
|
||||
# and sibling ORM objects. Owned sessions roll themselves back.
|
||||
if payment_confirmed and invoice.status != "settlement_pending":
|
||||
try:
|
||||
async with create_session() as state_session:
|
||||
pending = await state_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status).in_(
|
||||
_RETRYABLE_INVOICE_STATUSES
|
||||
),
|
||||
)
|
||||
.values(status="settlement_pending")
|
||||
)
|
||||
await state_session.commit()
|
||||
if pending.rowcount == 1:
|
||||
_publish_invoice_value(
|
||||
invoice, "status", "settlement_pending"
|
||||
)
|
||||
except Exception as state_error:
|
||||
logger.critical(
|
||||
"Paid invoice reconciliation state could not be persisted",
|
||||
extra={"invoice_id": invoice.id, "error": str(state_error)},
|
||||
)
|
||||
if minted:
|
||||
logger.critical(
|
||||
"Invoice mint succeeded but DB finalization failed; reconciliation required",
|
||||
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
|
||||
)
|
||||
try:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(error, Exception):
|
||||
raise
|
||||
logger.error(f"Failed to check invoice payment: {error}")
|
||||
return False
|
||||
|
||||
|
||||
async def create_api_key_from_invoice(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> ApiKey:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
|
||||
def _is_quote_not_found(error: BaseException) -> bool:
|
||||
"""Check if the error indicates the mint no longer has this quote."""
|
||||
message = str(error)
|
||||
return bool(
|
||||
re.search(r"\bquote\s+not\s+found\b", message, re.IGNORECASE)
|
||||
and re.search(r"\bcode\s*:?\s*0\b", message, re.IGNORECASE)
|
||||
)
|
||||
|
||||
|
||||
def _is_outputs_already_signed(error: BaseException) -> bool:
|
||||
message = str(error)
|
||||
return bool(
|
||||
re.search(
|
||||
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
|
||||
message,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
|
||||
)
|
||||
|
||||
|
||||
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
|
||||
"""Return spendable wallet value minted by one Lightning quote."""
|
||||
return sum(
|
||||
proof.amount
|
||||
for proof in wallet.proofs
|
||||
if proof.mint_id == quote_id and not proof.reserved
|
||||
)
|
||||
|
||||
|
||||
async def _mint_invoice_quote(
|
||||
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
|
||||
) -> None:
|
||||
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
await wallet.load_proofs(reload=True)
|
||||
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
|
||||
return
|
||||
|
||||
try:
|
||||
await run_mint_operation(
|
||||
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
|
||||
op_name=f"invoice_mint_{invoice.purpose}",
|
||||
mint_url=mint_url,
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except Exception as error:
|
||||
if not _is_outputs_already_signed(error):
|
||||
raise
|
||||
|
||||
for keyset_id in wallet.keysets:
|
||||
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await wallet.load_proofs(reload=True)
|
||||
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if recovered < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice outputs were already signed but quote-linked recovery returned "
|
||||
f"{recovered} sats; expected at least {invoice.amount_sats}"
|
||||
) from error
|
||||
else:
|
||||
await wallet.load_proofs(reload=True)
|
||||
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if minted_amount < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice mint succeeded but quote-linked proofs total "
|
||||
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
|
||||
)
|
||||
|
||||
|
||||
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
return hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
|
||||
|
||||
async def _create_api_key_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> ApiKey:
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
api_key = ApiKey(
|
||||
hashed_key=hashed_key,
|
||||
balance=invoice.amount_sats * 1000, # Convert to msats
|
||||
hashed_key=_invoice_api_key_hash(invoice),
|
||||
balance=invoice.amount_sats * 1000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url=settings.primary_mint,
|
||||
refund_mint_url=mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
)
|
||||
|
||||
session.add(api_key)
|
||||
await session.flush()
|
||||
|
||||
return api_key
|
||||
|
||||
|
||||
async def topup_api_key_from_invoice(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
async def _topup_api_key_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
|
||||
|
||||
if not invoice.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
|
||||
api_key = await session.get(ApiKey, invoice.api_key_hash)
|
||||
if not api_key:
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
|
||||
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
raise ValueError("Associated API key not found")
|
||||
|
||||
api_key.balance += invoice.amount_sats * 1000 # Convert to msats
|
||||
await session.flush()
|
||||
|
||||
async def _finalize_invoice_settlement(
|
||||
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Atomically fence and apply one invoice credit in the provided owned session."""
|
||||
api_key_hash = (
|
||||
_invoice_api_key_hash(invoice)
|
||||
if invoice.purpose == "create"
|
||||
else invoice.api_key_hash
|
||||
)
|
||||
claim = await session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(col(LightningInvoice.id) == invoice.id)
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
|
||||
)
|
||||
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False, None
|
||||
|
||||
if invoice.purpose == "create":
|
||||
await _create_api_key_record(invoice, session)
|
||||
elif invoice.purpose == "topup":
|
||||
await _topup_api_key_record(invoice, session)
|
||||
else:
|
||||
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
|
||||
await session.commit()
|
||||
return True, api_key_hash
|
||||
|
||||
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 5
|
||||
async def _reload_invoice_view(
|
||||
invoice: LightningInvoice, _caller_session: AsyncSession
|
||||
) -> None:
|
||||
"""Publish committed invoice state without touching the caller transaction."""
|
||||
async with create_session() as reload_session:
|
||||
stored = await reload_session.get(LightningInvoice, invoice.id)
|
||||
if stored is None:
|
||||
return
|
||||
status = stored.status
|
||||
paid_at = stored.paid_at
|
||||
api_key_hash = stored.api_key_hash
|
||||
await reload_session.commit()
|
||||
_publish_invoice_value(invoice, "status", status)
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
|
||||
|
||||
async def _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice: LightningInvoice,
|
||||
caller_session: AsyncSession,
|
||||
definitively_unpaid: bool,
|
||||
) -> bool:
|
||||
"""Expire one overdue unpaid invoice without overwriting concurrent settlement."""
|
||||
if (
|
||||
not definitively_unpaid
|
||||
or invoice.status != "pending"
|
||||
or int(time.time()) <= invoice.expires_at
|
||||
):
|
||||
return False
|
||||
|
||||
async with create_session() as expiry_session:
|
||||
expired = await expiry_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="expired")
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
await expiry_session.commit()
|
||||
|
||||
if expired.rowcount == 1:
|
||||
_publish_invoice_value(invoice, "status", "expired")
|
||||
return True
|
||||
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
|
||||
async def _credit_topup_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
await _topup_api_key_record(invoice, session)
|
||||
|
||||
|
||||
# Nutshell mints throttle Lightning backend lookups to once per 10s per
|
||||
# quote, so polling faster just burns the global request budget for nothing.
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 10
|
||||
INVOICE_WATCH_BATCH_LIMIT = 100
|
||||
|
||||
|
||||
async def periodic_invoice_watcher() -> None:
|
||||
"""Background task: detect paid Lightning invoices and credit balances.
|
||||
async def _process_invoice_watch_batch(session: AsyncSession) -> None:
|
||||
result = await session.exec(
|
||||
select(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
|
||||
)
|
||||
.limit(INVOICE_WATCH_BATCH_LIMIT)
|
||||
)
|
||||
for invoice in result.all():
|
||||
try:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Invoice watcher failed for invoice",
|
||||
extra={"invoice_id": invoice.id, "error": str(e)},
|
||||
)
|
||||
|
||||
Removes the need for clients to poll the status endpoint after paying.
|
||||
"""
|
||||
|
||||
async def periodic_invoice_watcher() -> None:
|
||||
"""Background task: detect paid Lightning invoices and credit balances."""
|
||||
while True:
|
||||
try:
|
||||
async with create_session() as session:
|
||||
now = int(time.time())
|
||||
result = await session.exec(
|
||||
select(LightningInvoice)
|
||||
.where(
|
||||
LightningInvoice.status == "pending",
|
||||
col(LightningInvoice.expires_at) > now,
|
||||
)
|
||||
.limit(INVOICE_WATCH_BATCH_LIMIT)
|
||||
)
|
||||
pending = result.all()
|
||||
for invoice in pending:
|
||||
try:
|
||||
await check_invoice_payment(invoice, session)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Invoice watcher failed for invoice",
|
||||
extra={"invoice_id": invoice.id, "error": str(e)},
|
||||
)
|
||||
await _process_invoice_watch_batch(session)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
+343
@@ -0,0 +1,343 @@
|
||||
"""Shared policy for bounded, rate-aware Cashu mint API operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any, AsyncGenerator, Awaitable, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
MINT_TRANSPORT_EXCEPTIONS: tuple[type[BaseException], ...] = (
|
||||
httpx.NetworkError,
|
||||
httpx.TimeoutException,
|
||||
ConnectionError,
|
||||
socket.gaierror,
|
||||
asyncio.TimeoutError,
|
||||
)
|
||||
|
||||
MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
|
||||
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
|
||||
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
|
||||
|
||||
_fail_fast_depth: ContextVar[int] = ContextVar("mint_fail_fast_depth", default=0)
|
||||
|
||||
|
||||
class MintRateLimitedError(httpx.HTTPStatusError):
|
||||
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
|
||||
|
||||
|
||||
class MintCooldownError(Exception):
|
||||
"""A mint is cooling down and this operation must not wait."""
|
||||
|
||||
def __init__(self, mint_url: str, retry_after_seconds: float):
|
||||
self.mint_url = mint_url
|
||||
self.retry_after_seconds = max(0.0, retry_after_seconds)
|
||||
super().__init__(
|
||||
f"Mint {mint_url} is cooling down; retry after "
|
||||
f"{self.retry_after_seconds:.2f}s"
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def fail_fast_mint_operations() -> AsyncGenerator[None, None]:
|
||||
"""Make mint cooldown/probe waits fail fast in the current task.
|
||||
|
||||
Wallet mutation code holds a process-wide file lock. It enters this scope so
|
||||
an existing mint cooldown can never turn that lock into a multi-hour wait.
|
||||
"""
|
||||
|
||||
token = _fail_fast_depth.set(_fail_fast_depth.get() + 1)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_fail_fast_depth.reset(token)
|
||||
|
||||
|
||||
class MintRateGuard:
|
||||
"""Limit concurrency and remember per-mint cooldown/probe state."""
|
||||
|
||||
_guards: dict[str, "MintRateGuard"] = {}
|
||||
|
||||
@classmethod
|
||||
def get(cls, mint_url: str) -> "MintRateGuard":
|
||||
concurrency = settings.mint_max_concurrency
|
||||
guard = cls._guards.get(mint_url)
|
||||
if guard is None or guard._max_concurrency != concurrency:
|
||||
previous = guard
|
||||
guard = cls(mint_url, concurrency)
|
||||
if previous is not None:
|
||||
# Concurrency changed at runtime: keep the live cooldown/backoff
|
||||
# state so an active 429 cooldown is not silently discarded.
|
||||
guard._cooldown_until = previous._cooldown_until
|
||||
guard._cooldown_reason = previous._cooldown_reason
|
||||
guard._consecutive_rate_limits = previous._consecutive_rate_limits
|
||||
guard._needs_probe = previous._needs_probe
|
||||
cls._guards[mint_url] = guard
|
||||
return guard
|
||||
|
||||
def __init__(self, mint_url: str, max_concurrency: int):
|
||||
self._mint_url = mint_url
|
||||
self._max_concurrency = max_concurrency
|
||||
self._semaphore = (
|
||||
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
|
||||
)
|
||||
self._cooldown_until = 0.0
|
||||
self._cooldown_reason: str | None = None
|
||||
self._consecutive_rate_limits = 0
|
||||
self._needs_probe = False
|
||||
self._probe_lock = asyncio.Lock()
|
||||
|
||||
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
|
||||
deadline = time.monotonic() + max(0.0, delay)
|
||||
if deadline >= self._cooldown_until:
|
||||
self._cooldown_until = deadline
|
||||
if reason is not None:
|
||||
self._cooldown_reason = reason
|
||||
elif self._cooldown_reason is None and reason is not None:
|
||||
self._cooldown_reason = reason
|
||||
self._needs_probe = True
|
||||
|
||||
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
|
||||
remaining = self.cooldown_remaining()
|
||||
if remaining > 0 and self._cooldown_reason == "rate_limited":
|
||||
minimum = min(
|
||||
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
|
||||
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
|
||||
)
|
||||
if minimum > remaining:
|
||||
self.apply_cooldown(minimum, reason="rate_limited")
|
||||
return minimum
|
||||
return remaining
|
||||
|
||||
self._consecutive_rate_limits += 1
|
||||
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
|
||||
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
|
||||
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
|
||||
self.apply_cooldown(delay, reason="rate_limited")
|
||||
return delay
|
||||
|
||||
def cooldown_remaining(self) -> float:
|
||||
return max(0.0, self._cooldown_until - time.monotonic())
|
||||
|
||||
def cooldown_reason(self) -> str | None:
|
||||
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
|
||||
|
||||
def _raise_if_wait_forbidden(self) -> None:
|
||||
remaining = self.cooldown_remaining()
|
||||
if _fail_fast_depth.get() and remaining > 0:
|
||||
raise MintCooldownError(self._mint_url, remaining)
|
||||
|
||||
async def _wait_for_cooldown(self) -> None:
|
||||
while True:
|
||||
self._raise_if_wait_forbidden()
|
||||
deadline = self._cooldown_until
|
||||
wait = max(0.0, deadline - time.monotonic())
|
||||
if wait <= 0:
|
||||
return
|
||||
logger.debug(
|
||||
"Mint rate guard: cooling down",
|
||||
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
if self._cooldown_until <= deadline:
|
||||
return
|
||||
|
||||
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
|
||||
await self._wait_for_cooldown()
|
||||
logger.info(
|
||||
"Mint cooldown ended; sending one probe request",
|
||||
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
|
||||
)
|
||||
try:
|
||||
result = await factory()
|
||||
except Exception as error:
|
||||
if is_mint_rate_limited(error):
|
||||
retry_after = None
|
||||
if isinstance(error, httpx.HTTPStatusError):
|
||||
retry_after = parse_retry_after(error.response.headers)
|
||||
self.apply_rate_limit_cooldown(retry_after)
|
||||
else:
|
||||
self.apply_cooldown(1.0)
|
||||
logger.warning(
|
||||
"Mint cooldown probe failed",
|
||||
extra={
|
||||
"event": "mint_cooldown_probe_failed",
|
||||
"mint_url": self._mint_url,
|
||||
"error": str(error),
|
||||
"error_type": type(error).__name__,
|
||||
"cooldown_seconds": round(self.cooldown_remaining(), 2),
|
||||
"consecutive_rate_limits": self._consecutive_rate_limits,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
self._needs_probe = False
|
||||
self._cooldown_until = 0.0
|
||||
self._cooldown_reason = None
|
||||
self._consecutive_rate_limits = 0
|
||||
logger.info(
|
||||
"Mint cooldown probe succeeded; restoring normal concurrency",
|
||||
extra={
|
||||
"event": "mint_cooldown_probe_succeeded",
|
||||
"mint_url": self._mint_url,
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
|
||||
while True:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self._needs_probe or self.cooldown_remaining() > 0:
|
||||
if _fail_fast_depth.get() and self._probe_lock.locked():
|
||||
raise MintCooldownError(self._mint_url, self.cooldown_remaining())
|
||||
async with self._probe_lock:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self.cooldown_remaining() > 0:
|
||||
self._needs_probe = True
|
||||
if self._needs_probe:
|
||||
return await self._run_probe(factory)
|
||||
continue
|
||||
|
||||
if self._semaphore is None:
|
||||
return await factory()
|
||||
async with self._semaphore:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self._needs_probe:
|
||||
continue
|
||||
return await factory()
|
||||
|
||||
|
||||
def mint_cooldown_remaining(mint_url: str) -> float:
|
||||
return MintRateGuard.get(mint_url).cooldown_remaining()
|
||||
|
||||
|
||||
def mint_cooldown_reason(mint_url: str) -> str | None:
|
||||
return MintRateGuard.get(mint_url).cooldown_reason()
|
||||
|
||||
|
||||
def is_mint_rate_limited(error: BaseException) -> bool:
|
||||
"""Return whether an exception chain represents HTTP 429/cooldown."""
|
||||
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
if isinstance(current, MintCooldownError):
|
||||
return True
|
||||
if isinstance(current, httpx.HTTPStatusError):
|
||||
if current.response.status_code == 429:
|
||||
return True
|
||||
current = current.__cause__ or current.__context__
|
||||
return False
|
||||
|
||||
|
||||
def parse_retry_after(headers: Any) -> float | None:
|
||||
raw = headers.get("retry-after") or headers.get("Retry-After")
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def run_mint_operation(
|
||||
factory: Callable[[], Awaitable[Any]],
|
||||
*,
|
||||
op_name: str = "mint_operation",
|
||||
mint_url: str = "",
|
||||
retry_timeouts: bool = True,
|
||||
retry_on_rate_limit: bool = True,
|
||||
) -> Any:
|
||||
"""Run one mint operation with bounded concurrency and adaptive cooldown."""
|
||||
|
||||
guard = MintRateGuard.get(mint_url) if mint_url else None
|
||||
timeout = settings.mint_operation_timeout_seconds
|
||||
max_attempts = settings.mint_retry_max_attempts + 1
|
||||
|
||||
async def timed_factory() -> Any:
|
||||
if timeout > 0:
|
||||
return await asyncio.wait_for(factory(), timeout=timeout)
|
||||
return await factory()
|
||||
|
||||
async def invoke() -> Any:
|
||||
if guard is not None:
|
||||
return await guard.run(timed_factory)
|
||||
return await timed_factory()
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
return await invoke()
|
||||
except MintCooldownError:
|
||||
raise
|
||||
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
|
||||
if retry_timeouts and attempt < max_attempts - 1:
|
||||
backoff = (2**attempt) + (time.monotonic() % 1.0)
|
||||
logger.warning(
|
||||
"Mint operation timed out, retrying",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"backoff_seconds": round(backoff, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
raise httpx.TimeoutException(
|
||||
f"{op_name} timed out (attempts: {attempt + 1})"
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
if not is_mint_rate_limited(exc):
|
||||
raise
|
||||
|
||||
backoff = (2**attempt) + (time.monotonic() % 1.0)
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
retry_after = parse_retry_after(exc.response.headers)
|
||||
if retry_after is not None:
|
||||
backoff = max(retry_after, backoff)
|
||||
cooldown = backoff
|
||||
if guard is not None:
|
||||
cooldown = guard.apply_rate_limit_cooldown(backoff)
|
||||
|
||||
if not retry_on_rate_limit:
|
||||
logger.warning(
|
||||
"Mint rate-limited, skipping retries for fallback",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"consecutive_rate_limits": guard._consecutive_rate_limits
|
||||
if guard is not None
|
||||
else attempt + 1,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
if attempt >= max_attempts - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
"Mint rate-limited, applying cooldown",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"consecutive_rate_limits": guard._consecutive_rate_limits
|
||||
if guard is not None
|
||||
else attempt + 1,
|
||||
},
|
||||
)
|
||||
if guard is None:
|
||||
await asyncio.sleep(cooldown)
|
||||
|
||||
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
|
||||
@@ -183,6 +183,28 @@ async def calculate_cost(
|
||||
cost_details.get("output_cost")
|
||||
or cost_details.get("upstream_inference_completions_cost")
|
||||
)
|
||||
cache_pricing_rates: tuple[float, float, float, float] | None = None
|
||||
if cache_read_tokens > 0 or cache_creation_tokens > 0:
|
||||
try:
|
||||
cache_pricing_rates = _get_pricing_rates(
|
||||
response_data, model_obj, provider_fee
|
||||
)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"Cache pricing unavailable for USD cost breakdown; "
|
||||
"leaving cache cost components unknown",
|
||||
extra={"model": response_data.get("model", "unknown")},
|
||||
)
|
||||
if cache_pricing_rates is None and settings.fixed_pricing:
|
||||
fixed_input_rate = (
|
||||
float(settings.fixed_per_1k_input_tokens) * 1000.0
|
||||
)
|
||||
cache_pricing_rates = (
|
||||
fixed_input_rate,
|
||||
float(settings.fixed_per_1k_output_tokens) * 1000.0,
|
||||
fixed_input_rate,
|
||||
fixed_input_rate,
|
||||
)
|
||||
return _calculate_from_usd_cost(
|
||||
usd_cost,
|
||||
input_usd,
|
||||
@@ -193,6 +215,7 @@ async def calculate_cost(
|
||||
output_tokens,
|
||||
response_data,
|
||||
provider_fee,
|
||||
cache_pricing_rates,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
@@ -451,6 +474,7 @@ def _calculate_from_usd_cost(
|
||||
output_tokens: int,
|
||||
response_data: dict,
|
||||
provider_fee: float | None,
|
||||
pricing_rates: tuple[float, float, float, float] | None = None,
|
||||
) -> CostData:
|
||||
"""Calculate cost from USD figures, deriving input/output split from tokens."""
|
||||
if provider_fee is None:
|
||||
@@ -460,15 +484,20 @@ def _calculate_from_usd_cost(
|
||||
output_usd = output_usd * provider_fee
|
||||
sats_per_usd = 1.0 / sats_usd_price()
|
||||
cost_in_sats = usd_cost * sats_per_usd
|
||||
cost_in_msats = math.ceil(cost_in_sats * 1000)
|
||||
raw_cost_msats = cost_in_sats * 1000
|
||||
cost_in_msats = math.ceil(raw_cost_msats)
|
||||
raw_input_msats = 0.0
|
||||
|
||||
if input_usd > 0 or output_usd > 0:
|
||||
# The total is the authoritative billed amount. Allocating that integer
|
||||
# total proportionally avoids losing sub-millisatoshi remainders when
|
||||
# input and output components are each truncated independently.
|
||||
component_usd = input_usd + output_usd
|
||||
input_msats = math.floor(cost_in_msats * input_usd / component_usd)
|
||||
output_msats = cost_in_msats - input_msats
|
||||
# Match the token-priced path: truncate the visible output component
|
||||
# and assign the authoritative total's rounding remainder to input.
|
||||
output_msats = math.floor(cost_in_msats * output_usd / component_usd)
|
||||
input_msats = cost_in_msats - output_msats
|
||||
raw_input_msats = raw_cost_msats * input_usd / component_usd
|
||||
else:
|
||||
effective_input_tokens = (
|
||||
input_tokens + cache_read_tokens + cache_creation_tokens
|
||||
@@ -480,6 +509,38 @@ def _calculate_from_usd_cost(
|
||||
else 0
|
||||
)
|
||||
output_msats = cost_in_msats - input_msats
|
||||
raw_input_msats = (
|
||||
raw_cost_msats * effective_input_tokens / total_tokens
|
||||
if total_tokens > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
# Preserve the same cache-rate ratios as the token-priced path while the
|
||||
# upstream USD total remains authoritative. Cache values are informational
|
||||
# subcomponents of the inclusive input cost.
|
||||
cache_read_msats = 0
|
||||
cache_creation_msats = 0
|
||||
if pricing_rates is not None:
|
||||
input_rate, _, cache_read_rate, cache_creation_rate = pricing_rates
|
||||
regular_weight = input_tokens * input_rate
|
||||
cache_read_weight = cache_read_tokens * cache_read_rate
|
||||
cache_creation_weight = cache_creation_tokens * cache_creation_rate
|
||||
total_input_weight = (
|
||||
regular_weight + cache_read_weight + cache_creation_weight
|
||||
)
|
||||
if total_input_weight > 0:
|
||||
cache_read_msats = int(
|
||||
round(
|
||||
raw_input_msats * cache_read_weight / total_input_weight,
|
||||
3,
|
||||
)
|
||||
)
|
||||
cache_creation_msats = int(
|
||||
round(
|
||||
raw_input_msats * cache_creation_weight / total_input_weight,
|
||||
3,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Using cost from usage data/details",
|
||||
@@ -487,6 +548,8 @@ def _calculate_from_usd_cost(
|
||||
"usd_cost": usd_cost,
|
||||
"cost_in_sats": cost_in_sats,
|
||||
"cost_in_msats": cost_in_msats,
|
||||
"cache_read_msats": cache_read_msats,
|
||||
"cache_creation_msats": cache_creation_msats,
|
||||
"model": response_data.get("model", "unknown"),
|
||||
},
|
||||
)
|
||||
@@ -501,8 +564,8 @@ def _calculate_from_usd_cost(
|
||||
output_tokens=output_tokens,
|
||||
cache_read_input_tokens=cache_read_tokens,
|
||||
cache_creation_input_tokens=cache_creation_tokens,
|
||||
cache_read_msats=0,
|
||||
cache_creation_msats=0,
|
||||
cache_read_msats=cache_read_msats,
|
||||
cache_creation_msats=cache_creation_msats,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from ..wallet import deserialize_token_from_string
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
|
||||
if x_cashu := headers.get("x-cashu", None):
|
||||
cashu_token = x_cashu
|
||||
@@ -243,7 +242,7 @@ async def calculate_discounted_max_cost(
|
||||
},
|
||||
)
|
||||
|
||||
return max(0, adjusted)
|
||||
return max(settings.min_request_msat, adjusted)
|
||||
|
||||
|
||||
def estimate_tokens(messages: list) -> int:
|
||||
|
||||
+63
-14
@@ -1,16 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from typing import TypedDict
|
||||
|
||||
import httpx
|
||||
from cashu.core.base import MeltQuoteState
|
||||
from cashu.wallet.wallet import Proof, Wallet
|
||||
|
||||
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
|
||||
# very slow mint can block a melt (and any caller, e.g. the payout loop)
|
||||
# indefinitely. Bound it here so callers fail instead of hanging forever.
|
||||
MELT_TIMEOUT_SECONDS = 60
|
||||
from ..mint import (
|
||||
MINT_TRANSPORT_EXCEPTIONS,
|
||||
is_mint_rate_limited,
|
||||
run_mint_operation,
|
||||
)
|
||||
|
||||
try:
|
||||
from bech32 import bech32_decode, convertbits # type: ignore
|
||||
@@ -31,6 +32,15 @@ class LNURLError(Exception):
|
||||
"""LNURL related errors."""
|
||||
|
||||
|
||||
class MeltOutcomeAmbiguousError(LNURLError):
|
||||
"""A melt was dispatched but its final outcome could not be confirmed.
|
||||
|
||||
Callers must NOT treat this as a clean failure: the payment may still
|
||||
settle, so debits backing it must be kept until reconciliation confirms
|
||||
the true outcome.
|
||||
"""
|
||||
|
||||
|
||||
async def decode_lnurl(lnurl: str) -> str:
|
||||
"""Decode LNURL to get the actual URL.
|
||||
|
||||
@@ -221,23 +231,62 @@ async def raw_send_to_lnurl(
|
||||
lnurl_data["callback_url"], final_amount
|
||||
)
|
||||
|
||||
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
|
||||
melt_quote_resp = await run_mint_operation(
|
||||
lambda: wallet.melt_quote(invoice=bolt11_invoice),
|
||||
op_name="lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
|
||||
if amount:
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
try:
|
||||
_ = await asyncio.wait_for(
|
||||
wallet.melt(
|
||||
melt_response = await run_mint_operation(
|
||||
lambda: wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
),
|
||||
timeout=MELT_TIMEOUT_SECONDS,
|
||||
op_name="lnurl_melt",
|
||||
mint_url=str(wallet.url),
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except asyncio.TimeoutError as e:
|
||||
raise LNURLError(
|
||||
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
|
||||
) from e
|
||||
return final_amount
|
||||
except Exception as error:
|
||||
if is_mint_rate_limited(error):
|
||||
# Cooldown failures happen before dispatch, and HTTP 429 means the
|
||||
# mint rejected the request. Neither outcome may keep proofs
|
||||
# reserved as though a Lightning payment could still settle.
|
||||
await wallet.set_reserved_for_send(proofs, reserved=False)
|
||||
raise
|
||||
if not isinstance(error, MINT_TRANSPORT_EXCEPTIONS):
|
||||
raise
|
||||
melt_response = None
|
||||
melt_error: BaseException | None = error
|
||||
else:
|
||||
melt_error = None
|
||||
|
||||
if getattr(melt_response, "state", None) == MeltQuoteState.paid:
|
||||
return final_amount
|
||||
|
||||
try:
|
||||
quote = await run_mint_operation(
|
||||
lambda: wallet.get_melt_quote(melt_quote_resp.quote),
|
||||
op_name="reconcile_lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except Exception as reconciliation_error:
|
||||
raise MeltOutcomeAmbiguousError(
|
||||
"Melt outcome is ambiguous; quote reconciliation failed and proofs "
|
||||
"must not be retried"
|
||||
) from reconciliation_error
|
||||
|
||||
if quote is not None and quote.state == MeltQuoteState.paid:
|
||||
return final_amount
|
||||
|
||||
state = getattr(getattr(quote, "state", None), "value", "unknown")
|
||||
raise MeltOutcomeAmbiguousError(
|
||||
"Melt outcome is ambiguous; proofs must not be retried "
|
||||
f"(quote_state={state})"
|
||||
) from melt_error
|
||||
|
||||
@@ -455,7 +455,9 @@ async def _update_sats_pricing_once() -> None:
|
||||
for m in upstream.get_cached_models()
|
||||
]
|
||||
upstream._models_cache = updated_models
|
||||
upstream._models_by_id = {m.forwarded_model_id or m.id: m for m in updated_models}
|
||||
upstream._models_by_id = {
|
||||
m.forwarded_model_id or m.id: m for m in updated_models
|
||||
}
|
||||
updated_count += len(updated_models)
|
||||
|
||||
if updated_count > 0:
|
||||
@@ -510,9 +512,7 @@ class ModelTestRequest(V2BaseModel):
|
||||
request_data: dict
|
||||
|
||||
|
||||
@models_router.post(
|
||||
"/api/models/test", dependencies=[Depends(_require_admin_api)]
|
||||
)
|
||||
@models_router.post("/api/models/test", dependencies=[Depends(_require_admin_api)])
|
||||
async def test_model(
|
||||
payload: ModelTestRequest,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
@@ -595,6 +595,37 @@ async def test_model(
|
||||
}
|
||||
|
||||
|
||||
@models_router.get("/v1/models/paths")
|
||||
@models_router.get("/v1/models/paths/", include_in_schema=False)
|
||||
async def model_paths() -> dict:
|
||||
"""All models with every upstream provider path they are reachable through."""
|
||||
from ..upstream.model_paths import get_all_model_paths
|
||||
|
||||
return await get_all_model_paths()
|
||||
|
||||
|
||||
@models_router.get("/v1/models/paths/model")
|
||||
@models_router.get("/v1/models/paths/model/", include_in_schema=False)
|
||||
async def model_paths_for_model(model_id: str) -> dict:
|
||||
"""Paths for a single model.
|
||||
|
||||
Uses a query parameter (``?model_id=...``) under a fully static route so
|
||||
model ids containing ``/`` (e.g. ``anthropic/claude-opus-4.6``) need no URL
|
||||
encoding and there is no dynamic-route ambiguity.
|
||||
"""
|
||||
from ..proxy import get_unique_models
|
||||
from ..upstream.model_paths import get_paths_for_model
|
||||
|
||||
result = await get_paths_for_model(model_id)
|
||||
if not result["data"]:
|
||||
advertised_ids = {
|
||||
model.forwarded_model_id or model.id for model in get_unique_models()
|
||||
}
|
||||
if model_id not in advertised_ids:
|
||||
raise HTTPException(status_code=404, detail="Model not found")
|
||||
return result
|
||||
|
||||
|
||||
@models_router.get("/v1/models")
|
||||
@models_router.get("/v1/models/", include_in_schema=False)
|
||||
@models_router.get("/models")
|
||||
|
||||
+90
-19
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
@@ -7,7 +8,13 @@ from fastapi.responses import Response, StreamingResponse
|
||||
from sqlmodel import select
|
||||
|
||||
from .algorithm import create_model_mappings
|
||||
from .auth import pay_for_request, revert_pay_for_request, validate_bearer_key
|
||||
from .auth import (
|
||||
ReservationSnapshot,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
validate_bearer_key,
|
||||
)
|
||||
from .core import get_logger
|
||||
from .core.db import (
|
||||
ApiKey,
|
||||
@@ -19,7 +26,6 @@ from .core.db import (
|
||||
)
|
||||
from .core.exceptions import UpstreamError
|
||||
from .core.not_found import build_not_found_response
|
||||
from .core.settings import settings
|
||||
from .payment.helpers import (
|
||||
calculate_discounted_max_cost,
|
||||
check_token_balance,
|
||||
@@ -43,6 +49,13 @@ _provider_map: dict[
|
||||
_unique_models: dict[str, Model] = {} # Unique model.id -> Model (no duplicates)
|
||||
|
||||
|
||||
async def _finish_read_transaction(session: AsyncSession) -> None:
|
||||
"""Release a read transaction without assuming a particular session mock."""
|
||||
commit_result = session.commit()
|
||||
if inspect.isawaitable(commit_result):
|
||||
await commit_result
|
||||
|
||||
|
||||
async def initialize_upstreams() -> None:
|
||||
"""Initialize upstream providers from database during application startup."""
|
||||
global _upstreams
|
||||
@@ -177,6 +190,19 @@ async def refresh_model_maps() -> None:
|
||||
disabled_model_keys=disabled_model_keys,
|
||||
)
|
||||
|
||||
# Keep model-path discovery in sync with admin mutations: disabling or
|
||||
# deleting a provider must stop advertising its paths immediately rather
|
||||
# than after the next timed refresh.
|
||||
from .upstream.model_paths import prune_model_paths_for_inactive_providers
|
||||
|
||||
try:
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
except Exception as e: # noqa: BLE001 - discovery sync must not break routing
|
||||
logger.warning(
|
||||
"Failed to prune model paths for inactive providers",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
|
||||
async def refresh_model_maps_periodically() -> None:
|
||||
"""Background task to refresh model maps every minute."""
|
||||
@@ -214,6 +240,20 @@ _API_PATH_PREFIXES = (
|
||||
@proxy_router.api_route("/{path:path}", methods=["GET", "POST"], response_model=None)
|
||||
async def proxy(
|
||||
request: Request, path: str, session: AsyncSession = Depends(get_session)
|
||||
) -> Response | StreamingResponse:
|
||||
"""Run proxy setup in a short request session, never across response streaming."""
|
||||
try:
|
||||
return await _proxy(request, path, session)
|
||||
finally:
|
||||
# FastAPI yield dependencies normally close after the response body is
|
||||
# sent. Close explicitly so a long stream cannot retain DB resources.
|
||||
close_result = session.close()
|
||||
if inspect.isawaitable(close_result):
|
||||
await close_result
|
||||
|
||||
|
||||
async def _proxy(
|
||||
request: Request, path: str, session: AsyncSession
|
||||
) -> Response | StreamingResponse:
|
||||
# GET requests must hit a known API prefix; otherwise return a 404 (HTML
|
||||
# for browsers, JSON for API clients). POST requests are always forwarded
|
||||
@@ -326,8 +366,6 @@ async def proxy(
|
||||
max_cost_for_model = await calculate_discounted_max_cost(
|
||||
_max_cost_for_model, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
# Ensure max_cost_for_model is at least the minimum allowed request cost
|
||||
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
|
||||
|
||||
check_token_balance(headers, request_body_dict, max_cost_for_model)
|
||||
|
||||
@@ -440,8 +478,13 @@ async def proxy(
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
)
|
||||
|
||||
reservation_snapshot: ReservationSnapshot | None = None
|
||||
if is_ehbp or request_body_dict:
|
||||
await pay_for_request(key, max_cost_for_model, session)
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
# Snapshot validation performs SELECTs after pay_for_request commits.
|
||||
# End that read transaction before waiting on upstream response headers.
|
||||
await _finish_read_transaction(session)
|
||||
|
||||
# Tracks request params already removed in response to upstream rejections,
|
||||
# shared across providers so a stripped param stays stripped on failover and
|
||||
@@ -461,16 +504,21 @@ async def proxy(
|
||||
candidate_max = await calculate_discounted_max_cost(
|
||||
candidate_max, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
candidate_max = max(candidate_max, settings.min_request_msat)
|
||||
if candidate_max > max_cost_for_model:
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
try:
|
||||
await pay_for_request(key, candidate_max, session)
|
||||
except HTTPException:
|
||||
if i == len(candidates) - 1:
|
||||
raise
|
||||
await pay_for_request(key, max_cost_for_model, session)
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
await _finish_read_transaction(session)
|
||||
continue
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
await _finish_read_transaction(session)
|
||||
max_cost_for_model = candidate_max
|
||||
|
||||
headers = upstream.prepare_headers(dict(request.headers))
|
||||
@@ -499,6 +547,7 @@ async def proxy(
|
||||
max_cost_for_model=max_cost_for_model,
|
||||
session=session,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
elif is_responses_api:
|
||||
response = await upstream.forward_responses_request(
|
||||
@@ -510,6 +559,7 @@ async def proxy(
|
||||
max_cost_for_model,
|
||||
session,
|
||||
model_obj,
|
||||
reservation_snapshot,
|
||||
)
|
||||
else:
|
||||
response = await upstream.forward_request(
|
||||
@@ -521,6 +571,7 @@ async def proxy(
|
||||
max_cost_for_model,
|
||||
session,
|
||||
model_obj,
|
||||
reservation_snapshot,
|
||||
)
|
||||
except UpstreamError:
|
||||
# Let the outer UpstreamError handler manage retry/revert
|
||||
@@ -537,7 +588,9 @@ async def proxy(
|
||||
"max_cost_for_model": max_cost_for_model,
|
||||
},
|
||||
)
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
raise
|
||||
|
||||
# Reactive recovery: some models reject one specific request
|
||||
@@ -606,7 +659,9 @@ async def proxy(
|
||||
continue
|
||||
|
||||
# 4xx error (user error), or other non-retryable error, or last provider failed
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
logger.warning(
|
||||
"Upstream request failed, revert payment "
|
||||
"(provider=%s model=%s status=%s path=%s)",
|
||||
@@ -638,8 +693,10 @@ async def proxy(
|
||||
"max_cost_for_model": max_cost_for_model,
|
||||
},
|
||||
)
|
||||
await asyncio.shield(
|
||||
revert_pay_for_request(key, session, max_cost_for_model)
|
||||
# The cancellation has been caught, so complete exact cleanup in
|
||||
# this task before the request-scoped session can be torn down.
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -659,7 +716,9 @@ async def proxy(
|
||||
|
||||
# If this was the last provider
|
||||
if i == len(candidates) - 1:
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
return create_upstream_error_response(e, request)
|
||||
|
||||
# Otherwise loop continues to next provider
|
||||
@@ -744,17 +803,29 @@ async def get_bearer_token_key(
|
||||
},
|
||||
)
|
||||
return key
|
||||
except Exception as e:
|
||||
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
|
||||
logger.error(
|
||||
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
|
||||
except HTTPException as error:
|
||||
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
|
||||
raw_error = detail.get("error")
|
||||
error_info = raw_error if isinstance(raw_error, dict) else {}
|
||||
logger.warning(
|
||||
"Bearer token rejected",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"status_code": error.status_code,
|
||||
"error_code": error_info.get("code"),
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"min_cost_msat": min_cost,
|
||||
"bearer_key_preview": key_preview,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"Bearer token validation failed",
|
||||
extra={
|
||||
"error_type": type(error).__name__,
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
+966
-12
File diff suppressed because it is too large
Load Diff
+400
-139
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,11 @@ from sqlmodel import col, update
|
||||
|
||||
from ..auth import (
|
||||
ROUTSTR_FEE_PERCENT,
|
||||
ReservationSnapshot,
|
||||
_claim_reservation_for_charge,
|
||||
_validate_reservation_snapshot,
|
||||
get_billing_key,
|
||||
get_reservation_snapshot,
|
||||
payments_logger,
|
||||
)
|
||||
from ..core import get_logger
|
||||
@@ -502,8 +506,14 @@ async def finalize_ehbp_actual_cost_payment(
|
||||
reserved_cost_for_model: int,
|
||||
model_id: str,
|
||||
cost_info: dict,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> None:
|
||||
"""Finalize an EHBP bearer request using clamped provider usage metrics."""
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, reservation, session)
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return
|
||||
reserved_cost_for_model = reservation.reserved_msats
|
||||
billing_key = await get_billing_key(key, session)
|
||||
key_hash = key.hashed_key
|
||||
billing_key_hash = billing_key.hashed_key
|
||||
@@ -606,6 +616,7 @@ async def finalize_ehbp_max_cost_payment(
|
||||
session: AsyncSession,
|
||||
max_cost_for_model: int,
|
||||
model_id: str,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> None:
|
||||
"""Finalize an EHBP bearer request by charging the reserved max cost.
|
||||
|
||||
@@ -613,6 +624,11 @@ async def finalize_ehbp_max_cost_payment(
|
||||
normal completion handlers, this intentionally charges the pre-reserved max
|
||||
cost and releases the reservation.
|
||||
"""
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, reservation, session)
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return
|
||||
max_cost_for_model = reservation.reserved_msats
|
||||
billing_key = await get_billing_key(key, session)
|
||||
key_hash = key.hashed_key
|
||||
billing_key_hash = billing_key.hashed_key
|
||||
@@ -766,6 +782,7 @@ async def forward_ehbp_request(
|
||||
max_cost_for_model: int,
|
||||
session: AsyncSession,
|
||||
model_obj: Model,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Forward an EHBP bearer-auth request and finalize billing.
|
||||
|
||||
@@ -883,7 +900,12 @@ async def forward_ehbp_request(
|
||||
# the requested model.
|
||||
billing_model = cost_info.pop("actual_model", None) or model_obj.id
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key, session, max_cost_for_model, billing_model, cost_info
|
||||
key,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
billing_model,
|
||||
cost_info,
|
||||
reservation_snapshot,
|
||||
)
|
||||
cost_data = {**cost_info, "total_usd": 0.0}
|
||||
else:
|
||||
@@ -897,7 +919,11 @@ async def forward_ehbp_request(
|
||||
},
|
||||
)
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
key, session, max_cost_for_model, model_obj.id
|
||||
key,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
model_obj.id,
|
||||
reservation_snapshot,
|
||||
)
|
||||
cost_data = {
|
||||
"total_msats": max_cost_for_model,
|
||||
|
||||
@@ -0,0 +1,819 @@
|
||||
"""Model-path discovery service.
|
||||
|
||||
Exposes every selectable upstream route a Routstr model is reachable through.
|
||||
This PR remains discovery-only: request-side routing will consume the opaque
|
||||
selectors in a follow-up.
|
||||
|
||||
A path is a standard percent-encoded query string containing the configured
|
||||
upstream URL, provider ID, client-visible model ID and, for an exact OpenRouter
|
||||
endpoint, its machine-readable tag::
|
||||
|
||||
url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=claude-sonnet-4
|
||||
url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=claude-sonnet-4&endpoint=google-vertex%2Fus
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.dialects.sqlite import insert
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlmodel import col, delete, select
|
||||
|
||||
from ..core.db import ModelPathRow, ModelRow, UpstreamProviderRow, create_session
|
||||
from ..core.logging import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from .base import BaseUpstreamProvider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Bound the per-model OpenRouter /endpoints fan-out so a provider with hundreds
|
||||
# of models does not open hundreds of concurrent requests every refresh.
|
||||
_OPENROUTER_CONCURRENCY = 5
|
||||
_OPENROUTER_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Rows inserted per statement during persist. Keeps each INSERT bounded while
|
||||
# avoiding per-row round-trips that hold SQLite's write lock for ~1s per cycle.
|
||||
_PERSIST_CHUNK_SIZE = 500
|
||||
|
||||
# Admin mutations enqueue provider IDs here instead of running OpenRouter's
|
||||
# per-model endpoint fan-out inside the request. One worker serializes refreshes
|
||||
# and coalesces repeated mutations for the same provider.
|
||||
_scheduled_provider_refresh_ids: set[int] = set()
|
||||
_scheduled_provider_refresh_task: asyncio.Task[None] | None = None
|
||||
|
||||
# Visibility key used across this module: routing carries the provider
|
||||
# dimension everywhere (ModelRow's primary key is (id, upstream_provider_id)),
|
||||
# so all model-id keyed maps here do too, lowercased like proxy.refresh_model_maps.
|
||||
ModelKey = tuple[str, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EndpointIdentity:
|
||||
"""Exact OpenRouter endpoint identity returned by ``/endpoints``."""
|
||||
|
||||
tag: str
|
||||
provider_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfiguredProviderIdentity:
|
||||
"""Public identity of one configured upstream provider."""
|
||||
|
||||
id: int
|
||||
slug: str
|
||||
provider_type: str
|
||||
base_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveredPath:
|
||||
"""One model route ready for persistence and API serialization."""
|
||||
|
||||
model_id: str
|
||||
path: str
|
||||
provider: ConfiguredProviderIdentity
|
||||
endpoint_tag: str | None = None
|
||||
endpoint_name: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderPathSnapshot:
|
||||
"""Refresh result plus model IDs whose prior rows must survive degradation."""
|
||||
|
||||
paths: tuple[DiscoveredPath, ...]
|
||||
preserve_model_ids: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def public_provider_url(base_url: str) -> str:
|
||||
"""Mask private IP addresses and URLs with explicit ports."""
|
||||
parsed = urlsplit(base_url)
|
||||
try:
|
||||
if parsed.port is not None:
|
||||
return "http://localhost"
|
||||
except ValueError:
|
||||
# An invalid explicit port must not accidentally leak through.
|
||||
return "http://localhost"
|
||||
|
||||
hostname = parsed.hostname
|
||||
if hostname is None:
|
||||
return base_url
|
||||
try:
|
||||
address = ipaddress.ip_address(hostname)
|
||||
except ValueError:
|
||||
return base_url
|
||||
return "http://localhost" if address.is_private else base_url
|
||||
|
||||
|
||||
def encode_model_path(
|
||||
base_url: str,
|
||||
provider_id: int,
|
||||
model_id: str,
|
||||
endpoint_tag: str | None = None,
|
||||
) -> str:
|
||||
"""Encode the complete upstream route selector advertised to clients."""
|
||||
components: list[tuple[str, str | int]] = [
|
||||
("url", base_url),
|
||||
("provider-id", provider_id),
|
||||
("model-id", model_id),
|
||||
]
|
||||
if endpoint_tag:
|
||||
components.append(("endpoint", endpoint_tag))
|
||||
return urlencode(components)
|
||||
|
||||
|
||||
def _make_http_client() -> httpx.AsyncClient:
|
||||
"""Client factory, separated so tests can substitute a mock transport."""
|
||||
return httpx.AsyncClient()
|
||||
|
||||
|
||||
def is_openrouter_base_url(base_url: str | None) -> bool:
|
||||
"""True when ``base_url`` points at OpenRouter.
|
||||
|
||||
Deliberately separate from ``BaseUpstreamProvider._upstream_accepts_cache_control``:
|
||||
that predicate also returns True for native Anthropic (correct for
|
||||
cache-control, wrong for OpenRouter endpoint discovery). This one keys only
|
||||
on the URL so a ``GenericUpstreamProvider`` aimed at OpenRouter is matched
|
||||
while native Anthropic is not.
|
||||
"""
|
||||
return "openrouter.ai" in (base_url or "")
|
||||
|
||||
|
||||
def exposed_model_id(model: object) -> str:
|
||||
"""Return exactly the ID advertised by ``/v1/models``.
|
||||
|
||||
A forwarded ID is already a public routable alias and must remain intact,
|
||||
including any slash. Without one, ``/v1/models`` exposes the base ID.
|
||||
"""
|
||||
forwarded = getattr(model, "forwarded_model_id", None)
|
||||
if forwarded:
|
||||
return forwarded
|
||||
return public_model_id(getattr(model, "id"))
|
||||
|
||||
|
||||
def public_model_id(model_id: str) -> str:
|
||||
"""Model id exposed by model-path API responses.
|
||||
|
||||
Uses the same rule as ``create_model_mappings.get_base_model_id`` and
|
||||
``resolve_model_alias`` — strip everything before the *first* slash — so
|
||||
the id shown here can be sent back to ``/v1/chat/completions`` verbatim.
|
||||
"""
|
||||
return model_id.split("/", 1)[1] if "/" in model_id else model_id
|
||||
|
||||
|
||||
def openrouter_author_slug(model: object) -> str | None:
|
||||
"""Return a canonical ``author/slug`` for the OpenRouter endpoints API.
|
||||
|
||||
Prefer ``canonical_slug``, then a slash-containing ``id``, then a
|
||||
slash-containing ``forwarded_model_id``. The forwarded id is exactly what
|
||||
the proxy sends upstream for admin-created alias rows (``base.py`` forwards
|
||||
``forwarded_model_id or id``), so it is a valid OpenRouter id when the
|
||||
bare ``id`` is a local alias with no slash.
|
||||
"""
|
||||
canonical = getattr(model, "canonical_slug", None)
|
||||
if canonical and "/" in canonical:
|
||||
return canonical
|
||||
model_id = getattr(model, "id", None)
|
||||
if model_id and "/" in model_id:
|
||||
return model_id
|
||||
forwarded = getattr(model, "forwarded_model_id", None)
|
||||
if forwarded and "/" in forwarded:
|
||||
return forwarded
|
||||
return None
|
||||
|
||||
|
||||
class _RefreshCycleState:
|
||||
"""Per-refresh shared state: fetch dedupe cache and rate-limit latch.
|
||||
|
||||
``endpoint_cache`` dedupes byte-identical ``/endpoints`` fetches when two
|
||||
providers point at the same OpenRouter base URL. ``rate_limited`` latches
|
||||
on the first 429 so the rest of the cycle stops hammering a throttled API;
|
||||
the whole provider result then degrades to "unknown" instead of an empty
|
||||
list, which preserves previously persisted rows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.endpoint_cache: dict[tuple[str, str], list[EndpointIdentity] | None] = {}
|
||||
self.rate_limited = False
|
||||
|
||||
|
||||
async def _fetch_openrouter_endpoint_subproviders(
|
||||
client: httpx.AsyncClient,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
author_slug: str,
|
||||
semaphore: asyncio.Semaphore,
|
||||
cycle: _RefreshCycleState,
|
||||
) -> list[EndpointIdentity] | None:
|
||||
"""Return exact endpoint identities for one model, or ``None`` when unknown.
|
||||
|
||||
``None`` (not ``[]``) signals a degraded fetch — network failure, rate
|
||||
limit, non-200, or an unparseable payload — so callers can distinguish
|
||||
"this model has no endpoints" from "we could not find out". Failures are
|
||||
logged and swallowed so one model never breaks the whole refresh.
|
||||
"""
|
||||
cache_key = (base_url, author_slug)
|
||||
if cache_key in cycle.endpoint_cache:
|
||||
return cycle.endpoint_cache[cache_key]
|
||||
if cycle.rate_limited:
|
||||
return None
|
||||
|
||||
url = f"{base_url.rstrip('/')}/models/{author_slug}/endpoints"
|
||||
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||||
result: list[EndpointIdentity] | None
|
||||
async with semaphore:
|
||||
try:
|
||||
resp = await client.get(
|
||||
url, headers=headers, timeout=_OPENROUTER_TIMEOUT_SECONDS
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - isolate per-model failures
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery request failed",
|
||||
extra={"author_slug": author_slug, "error": str(e)},
|
||||
)
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
|
||||
if resp.status_code == 429:
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery rate-limited; aborting cycle",
|
||||
extra={"author_slug": author_slug},
|
||||
)
|
||||
cycle.rate_limited = True
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery non-200",
|
||||
extra={"author_slug": author_slug, "status_code": resp.status_code},
|
||||
)
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
|
||||
try:
|
||||
payload = resp.json()
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
endpoints = data.get("endpoints") if isinstance(data, dict) else None
|
||||
if not isinstance(endpoints, list):
|
||||
raise ValueError("endpoints must be a list")
|
||||
identities: dict[str, EndpointIdentity] = {}
|
||||
for endpoint in endpoints:
|
||||
if not isinstance(endpoint, dict):
|
||||
continue
|
||||
tag = endpoint.get("tag")
|
||||
if not isinstance(tag, str) or not tag.strip():
|
||||
continue
|
||||
provider_name = endpoint.get("provider_name")
|
||||
identities.setdefault(
|
||||
tag,
|
||||
EndpointIdentity(
|
||||
tag=tag,
|
||||
provider_name=provider_name
|
||||
if isinstance(provider_name, str) and provider_name
|
||||
else None,
|
||||
),
|
||||
)
|
||||
if endpoints and not identities:
|
||||
raise ValueError("endpoints contain no usable tags")
|
||||
result = list(identities.values())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery bad payload",
|
||||
extra={"author_slug": author_slug, "error": str(e)},
|
||||
)
|
||||
result = None
|
||||
|
||||
cycle.endpoint_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
async def _load_model_visibility() -> tuple[
|
||||
dict[ModelKey, ModelRow],
|
||||
set[ModelKey],
|
||||
dict[int, ConfiguredProviderIdentity],
|
||||
]:
|
||||
"""Load the same DB model visibility inputs used by routing.
|
||||
|
||||
``refresh_model_maps`` builds routing from enabled providers, enabled DB
|
||||
override rows, and disabled model keys — all keyed on
|
||||
``(model_id.lower(), upstream_provider_id)`` because ``ModelRow``'s primary
|
||||
key is composite and the same id legitimately exists on several providers.
|
||||
Model-path discovery uses the same keying so disabling a model on one
|
||||
provider never hides it on another, and one provider's
|
||||
``forwarded_model_id`` alias is never applied to a different provider.
|
||||
"""
|
||||
async with create_session() as session:
|
||||
query = select(UpstreamProviderRow).options(
|
||||
selectinload(UpstreamProviderRow.models) # type: ignore[arg-type]
|
||||
)
|
||||
provider_rows = (await session.exec(query)).all()
|
||||
|
||||
overrides_by_key: dict[ModelKey, ModelRow] = {}
|
||||
disabled_model_keys: set[ModelKey] = set()
|
||||
provider_identities: dict[int, ConfiguredProviderIdentity] = {}
|
||||
|
||||
for provider in provider_rows:
|
||||
if not provider.enabled or provider.id is None:
|
||||
continue
|
||||
provider_identities[provider.id] = ConfiguredProviderIdentity(
|
||||
id=provider.id,
|
||||
slug=provider.slug or f"provider-{provider.id}",
|
||||
provider_type=provider.provider_type,
|
||||
base_url=public_provider_url(provider.base_url),
|
||||
)
|
||||
for model in provider.models:
|
||||
key = (model.id.lower(), provider.id)
|
||||
if model.enabled:
|
||||
overrides_by_key[key] = model
|
||||
else:
|
||||
disabled_model_keys.add(key)
|
||||
|
||||
return overrides_by_key, disabled_model_keys, provider_identities
|
||||
|
||||
|
||||
def _apply_model_visibility(
|
||||
upstream: BaseUpstreamProvider,
|
||||
overrides_by_key: dict[ModelKey, ModelRow] | None,
|
||||
disabled_model_keys: set[ModelKey] | None,
|
||||
) -> list[object]:
|
||||
"""Return provider models after DB disabled/override state is applied.
|
||||
|
||||
Only the identity fields (``id``, ``forwarded_model_id``,
|
||||
``canonical_slug``) matter for path discovery, so DB override rows are used
|
||||
directly rather than rebuilt into fully priced ``Model`` objects — the
|
||||
pricing pipeline costs ~0.7ms of event-loop CPU per row for data this
|
||||
module immediately discards.
|
||||
"""
|
||||
overrides_by_key = overrides_by_key or {}
|
||||
disabled_model_keys = disabled_model_keys or set()
|
||||
upstream_provider_id = getattr(upstream, "db_id", None)
|
||||
if not isinstance(upstream_provider_id, int):
|
||||
return [
|
||||
model
|
||||
for model in upstream.get_cached_models()
|
||||
if getattr(model, "enabled", True)
|
||||
]
|
||||
|
||||
visible_models: list[object] = []
|
||||
seen_model_ids: set[str] = set()
|
||||
|
||||
for model in upstream.get_cached_models():
|
||||
model_id = getattr(model, "id", "")
|
||||
key = (model_id.lower(), upstream_provider_id)
|
||||
if not getattr(model, "enabled", True) or key in disabled_model_keys:
|
||||
continue
|
||||
# Apply overrides only for this provider's own model row.
|
||||
override_row = overrides_by_key.get(key)
|
||||
visible: object = model if override_row is None else override_row
|
||||
visible_models.append(visible)
|
||||
seen_model_ids.add(model_id.lower())
|
||||
|
||||
# DB-only override rows for this provider with no cached counterpart.
|
||||
for (model_id_lower, provider_id), override_row in overrides_by_key.items():
|
||||
if provider_id != upstream_provider_id:
|
||||
continue
|
||||
if model_id_lower in seen_model_ids:
|
||||
continue
|
||||
visible_models.append(override_row)
|
||||
seen_model_ids.add(model_id_lower)
|
||||
|
||||
return visible_models
|
||||
|
||||
|
||||
async def _collect_provider_paths(
|
||||
upstream: BaseUpstreamProvider,
|
||||
provider_identity: ConfiguredProviderIdentity,
|
||||
overrides_by_key: dict[ModelKey, ModelRow] | None = None,
|
||||
disabled_model_keys: set[ModelKey] | None = None,
|
||||
cycle: _RefreshCycleState | None = None,
|
||||
) -> ProviderPathSnapshot:
|
||||
"""Collect selectable routes while marking model-level degraded fetches.
|
||||
|
||||
A failed OpenRouter lookup preserves only that model's prior rows. Other
|
||||
models in the same provider still refresh, so a partial outage cannot erase
|
||||
valid discovery data or freeze the entire provider snapshot.
|
||||
"""
|
||||
cycle = cycle or _RefreshCycleState()
|
||||
models = _apply_model_visibility(upstream, overrides_by_key, disabled_model_keys)
|
||||
|
||||
def _base_path(model: object) -> DiscoveredPath:
|
||||
model_id = exposed_model_id(model)
|
||||
return DiscoveredPath(
|
||||
model_id=model_id,
|
||||
path=encode_model_path(
|
||||
provider_identity.base_url, provider_identity.id, model_id
|
||||
),
|
||||
provider=provider_identity,
|
||||
)
|
||||
|
||||
if not is_openrouter_base_url(upstream.base_url):
|
||||
return ProviderPathSnapshot(paths=tuple(_base_path(model) for model in models))
|
||||
|
||||
if not (upstream.provider_type or "").strip():
|
||||
return ProviderPathSnapshot(paths=())
|
||||
|
||||
semaphore = asyncio.Semaphore(_OPENROUTER_CONCURRENCY)
|
||||
async with _make_http_client() as client:
|
||||
|
||||
async def _for_model(
|
||||
model: object,
|
||||
) -> tuple[list[DiscoveredPath], str | None]:
|
||||
model_id = exposed_model_id(model)
|
||||
author_slug = openrouter_author_slug(model)
|
||||
if not author_slug:
|
||||
return [_base_path(model)], None
|
||||
endpoints = await _fetch_openrouter_endpoint_subproviders(
|
||||
client,
|
||||
upstream.base_url,
|
||||
upstream.api_key,
|
||||
author_slug,
|
||||
semaphore,
|
||||
cycle,
|
||||
)
|
||||
if endpoints is None:
|
||||
return [], model_id
|
||||
paths = [_base_path(model)]
|
||||
paths.extend(
|
||||
DiscoveredPath(
|
||||
model_id=model_id,
|
||||
path=encode_model_path(
|
||||
provider_identity.base_url,
|
||||
provider_identity.id,
|
||||
model_id,
|
||||
endpoint.tag,
|
||||
),
|
||||
provider=provider_identity,
|
||||
endpoint_tag=endpoint.tag,
|
||||
endpoint_name=endpoint.provider_name,
|
||||
)
|
||||
for endpoint in endpoints
|
||||
)
|
||||
return paths, None
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_for_model(model) for model in models), return_exceptions=True
|
||||
)
|
||||
|
||||
paths: list[DiscoveredPath] = []
|
||||
preserve_model_ids: set[str] = set()
|
||||
for model, result in zip(models, results):
|
||||
if isinstance(result, BaseException):
|
||||
model_id = exposed_model_id(model)
|
||||
preserve_model_ids.add(model_id)
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery task errored",
|
||||
extra={"provider": upstream.provider_type, "error": str(result)},
|
||||
)
|
||||
continue
|
||||
model_paths, preserved_model_id = result
|
||||
paths.extend(model_paths)
|
||||
if preserved_model_id:
|
||||
preserve_model_ids.add(preserved_model_id)
|
||||
|
||||
return ProviderPathSnapshot(
|
||||
paths=tuple(paths), preserve_model_ids=frozenset(preserve_model_ids)
|
||||
)
|
||||
|
||||
|
||||
async def _persist_provider_paths(
|
||||
upstream_provider_id: int, snapshot: ProviderPathSnapshot
|
||||
) -> None:
|
||||
"""Replace refreshed rows while retaining model-level degraded snapshots."""
|
||||
unique_paths = list(
|
||||
{(path.model_id, path.path): path for path in snapshot.paths}.values()
|
||||
)
|
||||
now = int(time.time())
|
||||
async with create_session() as session:
|
||||
delete_stmt = delete(ModelPathRow).where(
|
||||
col(ModelPathRow.upstream_provider_id) == upstream_provider_id
|
||||
)
|
||||
if snapshot.preserve_model_ids:
|
||||
delete_stmt = delete_stmt.where(
|
||||
col(ModelPathRow.model_id).not_in(sorted(snapshot.preserve_model_ids))
|
||||
)
|
||||
await session.exec(delete_stmt) # type: ignore[call-overload]
|
||||
for start in range(0, len(unique_paths), _PERSIST_CHUNK_SIZE):
|
||||
chunk = unique_paths[start : start + _PERSIST_CHUNK_SIZE]
|
||||
values = [
|
||||
{
|
||||
"model_id": discovered.model_id,
|
||||
"path": discovered.path,
|
||||
"provider_slug": discovered.provider.slug,
|
||||
"provider_type": discovered.provider.provider_type,
|
||||
"endpoint_tag": discovered.endpoint_tag,
|
||||
"endpoint_name": discovered.endpoint_name,
|
||||
"upstream_provider_id": upstream_provider_id,
|
||||
"updated_at": now,
|
||||
}
|
||||
for discovered in chunk
|
||||
]
|
||||
insert_stmt = insert(ModelPathRow).values(values)
|
||||
await session.execute(
|
||||
insert_stmt.on_conflict_do_update(
|
||||
index_elements=["model_id", "path", "upstream_provider_id"],
|
||||
set_={
|
||||
"provider_slug": insert_stmt.excluded.provider_slug,
|
||||
"provider_type": insert_stmt.excluded.provider_type,
|
||||
"endpoint_tag": insert_stmt.excluded.endpoint_tag,
|
||||
"endpoint_name": insert_stmt.excluded.endpoint_name,
|
||||
"updated_at": insert_stmt.excluded.updated_at,
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def prune_model_paths_for_inactive_providers() -> None:
|
||||
"""Delete paths whose provider is no longer enabled in the database.
|
||||
|
||||
Called from ``refresh_model_maps`` so admin mutations (disable/delete
|
||||
provider) stop advertising a provider's paths immediately instead of
|
||||
waiting for the next timed refresh. Uses the DB as the source of truth, so
|
||||
it is safe at boot even before upstreams initialize.
|
||||
"""
|
||||
async with create_session() as session:
|
||||
enabled_ids = (
|
||||
await session.exec(
|
||||
select(UpstreamProviderRow.id).where(
|
||||
col(UpstreamProviderRow.enabled).is_(True)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
stmt = delete(ModelPathRow)
|
||||
if enabled_ids:
|
||||
stmt = stmt.where(
|
||||
col(ModelPathRow.upstream_provider_id).not_in(
|
||||
[pid for pid in enabled_ids if pid is not None]
|
||||
)
|
||||
)
|
||||
await session.exec(stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def refresh_model_paths(
|
||||
upstreams: list[BaseUpstreamProvider],
|
||||
) -> None:
|
||||
"""Recompute and persist model paths for every enabled provider.
|
||||
|
||||
One provider's failure is logged and isolated; it must not break the rest.
|
||||
A provider whose paths could not be determined this cycle keeps its
|
||||
previously persisted rows. An empty ``upstreams`` list (e.g. a failed
|
||||
``initialize_upstreams`` at boot) is treated as "unknown" and touches
|
||||
nothing.
|
||||
"""
|
||||
if not upstreams:
|
||||
logger.warning("Skipping model paths refresh: no live upstreams")
|
||||
return
|
||||
|
||||
(
|
||||
overrides_by_key,
|
||||
disabled_model_keys,
|
||||
provider_identities,
|
||||
) = await _load_model_visibility()
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
|
||||
cycle = _RefreshCycleState()
|
||||
for upstream in upstreams:
|
||||
if upstream.db_id is None or upstream.db_id not in provider_identities:
|
||||
continue
|
||||
try:
|
||||
snapshot = await _collect_provider_paths(
|
||||
upstream,
|
||||
provider_identity=provider_identities[upstream.db_id],
|
||||
overrides_by_key=overrides_by_key,
|
||||
disabled_model_keys=disabled_model_keys,
|
||||
cycle=cycle,
|
||||
)
|
||||
if snapshot.preserve_model_ids:
|
||||
logger.warning(
|
||||
"Some model paths are unknown; keeping their previous rows",
|
||||
extra={
|
||||
"provider": upstream.provider_type or upstream.base_url,
|
||||
"db_id": upstream.db_id,
|
||||
"preserved_models": len(snapshot.preserve_model_ids),
|
||||
},
|
||||
)
|
||||
await _persist_provider_paths(upstream.db_id, snapshot)
|
||||
except Exception as e: # noqa: BLE001 - isolate per-provider failures
|
||||
logger.error(
|
||||
"Failed to refresh model paths for provider",
|
||||
extra={
|
||||
"provider": upstream.provider_type or upstream.base_url,
|
||||
"db_id": upstream.db_id,
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def refresh_model_paths_for_provider(upstream_provider_id: int) -> None:
|
||||
"""Synchronize one provider when model-path discovery is enabled."""
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
return
|
||||
|
||||
from ..proxy import get_upstreams
|
||||
|
||||
matching = [
|
||||
upstream
|
||||
for upstream in get_upstreams()
|
||||
if upstream.db_id == upstream_provider_id
|
||||
]
|
||||
if matching:
|
||||
await refresh_model_paths(matching)
|
||||
else:
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
|
||||
|
||||
async def _drain_scheduled_provider_refreshes() -> None:
|
||||
"""Serialize and coalesce model-path refreshes scheduled by admin writes."""
|
||||
global _scheduled_provider_refresh_task
|
||||
|
||||
try:
|
||||
# Let mutations in the same event-loop turn collapse into one refresh.
|
||||
await asyncio.sleep(0)
|
||||
while _scheduled_provider_refresh_ids:
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
_scheduled_provider_refresh_ids.clear()
|
||||
return
|
||||
provider_id = min(_scheduled_provider_refresh_ids)
|
||||
_scheduled_provider_refresh_ids.remove(provider_id)
|
||||
try:
|
||||
await refresh_model_paths_for_provider(provider_id)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - background best effort
|
||||
logger.warning(
|
||||
"Failed to refresh model paths after admin mutation",
|
||||
extra={
|
||||
"upstream_provider_id": provider_id,
|
||||
"error": str(exc),
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
_scheduled_provider_refresh_task = None
|
||||
|
||||
|
||||
async def schedule_model_paths_refresh_for_provider(
|
||||
upstream_provider_id: int,
|
||||
) -> None:
|
||||
"""Queue a non-blocking, coalesced refresh after an admin mutation."""
|
||||
global _scheduled_provider_refresh_task
|
||||
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
return
|
||||
_scheduled_provider_refresh_ids.add(upstream_provider_id)
|
||||
if (
|
||||
_scheduled_provider_refresh_task is None
|
||||
or _scheduled_provider_refresh_task.done()
|
||||
):
|
||||
_scheduled_provider_refresh_task = asyncio.create_task(
|
||||
_drain_scheduled_provider_refreshes(),
|
||||
name="model-path-admin-refresh",
|
||||
)
|
||||
|
||||
|
||||
def _refresh_interval_seconds() -> int:
|
||||
"""Current interval, re-read every loop so runtime setting changes apply."""
|
||||
from ..core.settings import settings
|
||||
|
||||
if not getattr(settings, "enable_model_paths_refresh", True):
|
||||
return 0
|
||||
return int(getattr(settings, "model_paths_refresh_interval_seconds", 0) or 0)
|
||||
|
||||
|
||||
async def refresh_model_paths_periodically(
|
||||
upstreams_provider: (
|
||||
Callable[[], list[BaseUpstreamProvider]] | list[BaseUpstreamProvider]
|
||||
),
|
||||
) -> None:
|
||||
"""Background task mirroring ``refresh_upstreams_models_periodically``.
|
||||
|
||||
The interval and enable flag are re-read every iteration, so the refresh
|
||||
can be turned off (or on) and retuned without a restart. While disabled the
|
||||
task idles instead of exiting, so re-enabling takes effect.
|
||||
"""
|
||||
_DISABLED_POLL_SECONDS = 60.0
|
||||
|
||||
def _resolve_upstreams() -> list[BaseUpstreamProvider]:
|
||||
if callable(upstreams_provider):
|
||||
return upstreams_provider()
|
||||
return upstreams_provider
|
||||
|
||||
while True:
|
||||
interval = _refresh_interval_seconds()
|
||||
if interval <= 0:
|
||||
try:
|
||||
await asyncio.sleep(_DISABLED_POLL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
continue
|
||||
|
||||
try:
|
||||
await refresh_model_paths(_resolve_upstreams())
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(
|
||||
"Error in model paths refresh loop",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
try:
|
||||
jitter = max(0.0, float(interval) * 0.1)
|
||||
await asyncio.sleep(interval + random.uniform(0, jitter))
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
|
||||
|
||||
def _serialize_path(row: ModelPathRow) -> dict[str, Any]:
|
||||
endpoint = None
|
||||
if row.endpoint_tag or row.endpoint_name:
|
||||
endpoint = {"tag": row.endpoint_tag, "name": row.endpoint_name}
|
||||
return {
|
||||
"path": row.path,
|
||||
"provider": {
|
||||
"id": row.upstream_provider_id,
|
||||
"slug": row.provider_slug,
|
||||
"type": row.provider_type,
|
||||
},
|
||||
"endpoint": endpoint,
|
||||
}
|
||||
|
||||
|
||||
async def get_all_model_paths() -> dict:
|
||||
"""All models with their exact selectable routes."""
|
||||
async with create_session() as session:
|
||||
rows = (
|
||||
await session.exec(
|
||||
select(ModelPathRow).order_by(
|
||||
col(ModelPathRow.model_id),
|
||||
col(ModelPathRow.path),
|
||||
col(ModelPathRow.upstream_provider_id),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
seen_paths: dict[str, set[str]] = {}
|
||||
updated_at = 0
|
||||
for row in rows:
|
||||
updated_at = max(updated_at, row.updated_at)
|
||||
if row.path in seen_paths.setdefault(row.model_id, set()):
|
||||
continue
|
||||
seen_paths[row.model_id].add(row.path)
|
||||
grouped.setdefault(row.model_id, []).append(_serialize_path(row))
|
||||
data = [
|
||||
{
|
||||
"id": grouped_model_id,
|
||||
"paths": grouped[grouped_model_id],
|
||||
}
|
||||
for grouped_model_id in sorted(grouped)
|
||||
]
|
||||
return {"data": data, "updated_at": updated_at or None}
|
||||
|
||||
|
||||
async def get_paths_for_model(model_id: str) -> dict:
|
||||
"""Return paths for an advertised ID or its provider-prefixed alias."""
|
||||
|
||||
async def load_rows(session: AsyncSession, lookup_id: str) -> list[ModelPathRow]:
|
||||
return list(
|
||||
(
|
||||
await session.exec(
|
||||
select(ModelPathRow)
|
||||
.where(col(ModelPathRow.model_id) == lookup_id)
|
||||
.order_by(
|
||||
col(ModelPathRow.path),
|
||||
col(ModelPathRow.upstream_provider_id),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
async with create_session() as session:
|
||||
rows = await load_rows(session, model_id)
|
||||
if not rows:
|
||||
unprefixed_id = public_model_id(model_id)
|
||||
if unprefixed_id != model_id:
|
||||
rows = await load_rows(session, unprefixed_id)
|
||||
|
||||
seen: set[str] = set()
|
||||
paths: list[dict] = []
|
||||
updated_at = 0
|
||||
for row in rows:
|
||||
updated_at = max(updated_at, row.updated_at)
|
||||
if row.path in seen:
|
||||
continue
|
||||
seen.add(row.path)
|
||||
paths.append(_serialize_path(row))
|
||||
return {"data": paths, "updated_at": updated_at or None}
|
||||
@@ -441,7 +441,7 @@ class PPQAIUpstreamProvider(BaseUpstreamProvider):
|
||||
"""
|
||||
data = await self.check_balance()
|
||||
balance = data.get("balance")
|
||||
if isinstance(balance, (int, float)):
|
||||
if isinstance(balance, (int, float)) and not isinstance(balance, bool):
|
||||
return float(balance)
|
||||
return None
|
||||
|
||||
|
||||
+1640
-320
File diff suppressed because it is too large
Load Diff
@@ -203,8 +203,13 @@ class TestmintWallet:
|
||||
token_base64 = base64.urlsafe_b64encode(token_json.encode()).decode()
|
||||
return f"cashuA{token_base64}"
|
||||
|
||||
async def redeem_token(self, token: str) -> Tuple[int, str, str]:
|
||||
"""Redeem a Cashu token - compatible with wallet.recieve_token"""
|
||||
async def redeem_token(
|
||||
self,
|
||||
token: str,
|
||||
destination_mint: str | None = None,
|
||||
destination_unit: str | None = None,
|
||||
) -> Tuple[int, str, str]:
|
||||
"""Redeem a Cashu token - compatible with wallet.recieve_token."""
|
||||
if not self.wallet:
|
||||
await self.init()
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.payment.cost_calculation import CostData
|
||||
|
||||
@@ -23,7 +24,7 @@ def _make_key(balance: int, reserved: int) -> ApiKey:
|
||||
return ApiKey(
|
||||
hashed_key=f"test_{uuid.uuid4().hex}",
|
||||
balance=balance,
|
||||
reserved_balance=reserved,
|
||||
reserved_balance=0,
|
||||
total_spent=0,
|
||||
total_requests=1,
|
||||
)
|
||||
@@ -75,6 +76,8 @@ async def test_balance_never_negative_when_cost_exceeds_reservation(
|
||||
key = _make_key(balance=deducted_max_cost, reserved=deducted_max_cost)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
|
||||
|
||||
@@ -111,6 +114,8 @@ async def test_balance_floor_at_zero_on_overrun(
|
||||
key = _make_key(balance=500, reserved=500)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 50}}
|
||||
|
||||
@@ -152,6 +157,8 @@ async def test_full_cost_charged_when_balance_sufficient_for_overrun(
|
||||
key = _make_key(balance=2000, reserved=990)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
|
||||
|
||||
@@ -190,7 +197,11 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
"""Concurrent finalization with cost overruns must never produce negative balance."""
|
||||
import asyncio
|
||||
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import create_session
|
||||
|
||||
deducted_max_cost = 990
|
||||
@@ -216,12 +227,14 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
async with create_session() as session:
|
||||
key_to_reserve = await session.get(ApiKey, key_hash)
|
||||
assert key_to_reserve is not None
|
||||
reservations = []
|
||||
for _ in range(n_requests):
|
||||
await pay_for_request(key_to_reserve, deducted_max_cost, session)
|
||||
reservations.append(await get_reservation_snapshot(key_to_reserve, session))
|
||||
await session.refresh(key_to_reserve)
|
||||
|
||||
# Now finalize all concurrently with cost overrun
|
||||
async def finalize() -> None:
|
||||
async def finalize(reservation: ReservationSnapshot) -> None:
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
|
||||
@@ -230,7 +243,11 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
fresh_key = await session.get(ApiKey, key_hash)
|
||||
assert fresh_key is not None
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key, response_data, session, deducted_max_cost, None, None
|
||||
fresh_key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
# Patch once around the gather: entering the same patch target from
|
||||
@@ -240,7 +257,7 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await asyncio.gather(*[finalize() for _ in range(n_requests)])
|
||||
await asyncio.gather(*(finalize(r) for r in reservations))
|
||||
|
||||
async with create_session() as session:
|
||||
final_key = await session.get(ApiKey, key_hash)
|
||||
@@ -281,6 +298,8 @@ async def test_zero_free_balance_overrun_is_safe(
|
||||
key = _make_key(balance=1000, reserved=1000)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 100}}
|
||||
|
||||
@@ -319,7 +338,11 @@ async def test_parallel_requests_no_free_inference(
|
||||
"""Second parallel finalization must be charged even when first depleted free balance."""
|
||||
import asyncio
|
||||
|
||||
from routstr.auth import adjust_payment_for_tokens
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import create_session
|
||||
|
||||
deducted_max_cost = 100
|
||||
@@ -340,14 +363,18 @@ async def test_parallel_requests_no_free_inference(
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=starting_balance,
|
||||
reserved_balance=deducted_max_cost * 2, # both slots pre-reserved
|
||||
reserved_balance=0,
|
||||
total_spent=0,
|
||||
total_requests=2,
|
||||
)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
reservations = []
|
||||
for _ in range(2):
|
||||
await pay_for_request(key, deducted_max_cost, session)
|
||||
reservations.append(await get_reservation_snapshot(key, session))
|
||||
|
||||
async def finalize() -> None:
|
||||
async def finalize(reservation: ReservationSnapshot) -> None:
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 100},
|
||||
@@ -356,7 +383,11 @@ async def test_parallel_requests_no_free_inference(
|
||||
fresh_key = await session.get(ApiKey, key_hash)
|
||||
assert fresh_key is not None
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key, response_data, session, deducted_max_cost, None, None
|
||||
fresh_key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
# Patch once around the gather: entering the same patch target from two
|
||||
@@ -366,7 +397,7 @@ async def test_parallel_requests_no_free_inference(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await asyncio.gather(finalize(), finalize())
|
||||
await asyncio.gather(*(finalize(r) for r in reservations))
|
||||
|
||||
async with create_session() as session:
|
||||
final_key = await session.get(ApiKey, key_hash)
|
||||
|
||||
@@ -14,13 +14,17 @@ from unittest.mock import patch
|
||||
import httpx
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
from routstr.proxy import refresh_model_maps
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
CHEAP_BASE_URL = "https://cheap.example.com/v1"
|
||||
EXPENSIVE_BASE_URL = "https://expensive.example.com/v1"
|
||||
THIRD_BASE_URL = "https://third.example.com/v1"
|
||||
|
||||
|
||||
def _make_model(
|
||||
@@ -55,9 +59,7 @@ def _make_model(
|
||||
class _StaticProvider(BaseUpstreamProvider):
|
||||
"""Upstream provider with a fixed model catalog and no remote refresh."""
|
||||
|
||||
def __init__(
|
||||
self, base_url: str, api_key: str, fee: float, model: Model
|
||||
) -> None:
|
||||
def __init__(self, base_url: str, api_key: str, fee: float, model: Model) -> None:
|
||||
super().__init__(base_url, api_key, fee)
|
||||
self.provider_type = "custom"
|
||||
self._static_model = model
|
||||
@@ -98,7 +100,7 @@ async def dual_provider_maps(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.0,
|
||||
_make_model("provb/dual-model", 0.005, 0.010),
|
||||
_make_model("provb/dual-model", 0.005, 0.010, max_cost=100.0),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield cheap, expensive
|
||||
@@ -142,6 +144,7 @@ def _upstream_response(request: httpx.Request) -> httpx.Response:
|
||||
async def test_failover_serve_billed_at_serving_providers_rate(
|
||||
authenticated_client: AsyncClient,
|
||||
dual_provider_maps: tuple[_StaticProvider, _StaticProvider],
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A fallback serve is billed at the fallback's price, not the winner's.
|
||||
|
||||
@@ -199,6 +202,17 @@ async def test_failover_serve_billed_at_serving_providers_rate(
|
||||
# Billed at the serving provider's rate: 1000/1000*5000 + 500/1000*10000.
|
||||
assert payload["cost"]["total_msats"] == 10_000
|
||||
|
||||
# The fallback's larger max-cost envelope requires a replacement
|
||||
# reservation. The failed candidate is released, the serving candidate is
|
||||
# charged, and no request-owned reservation remains active.
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert sorted(record.status for record in records) == ["charged", "released"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def same_id_provider_maps(
|
||||
@@ -349,9 +363,7 @@ async def test_usd_cost_serve_carries_serving_providers_fee(
|
||||
if request.url.host == "cheap.example.com":
|
||||
return httpx.Response(
|
||||
502,
|
||||
content=json.dumps(
|
||||
{"error": {"message": "bad gateway"}}
|
||||
).encode(),
|
||||
content=json.dumps({"error": {"message": "bad gateway"}}).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
body = {
|
||||
@@ -478,6 +490,101 @@ async def test_failover_beyond_balance_envelope_is_rejected(
|
||||
assert [r.url.host for r in sent_requests] == ["cheap.example.com"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def three_candidate_child_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Second candidate cannot fit the child limit; third restores and serves."""
|
||||
first = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-first",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
|
||||
)
|
||||
too_large = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-too-large",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.002, 0.003, max_cost=100.0),
|
||||
)
|
||||
third = _StaticProvider(
|
||||
THIRD_BASE_URL,
|
||||
"key-third",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.003, 0.004, max_cost=50.0),
|
||||
)
|
||||
async for _ in _install_providers([first, too_large, third]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_child_failover_rolls_back_failed_larger_reserve_before_restoring(
|
||||
authenticated_client: AsyncClient,
|
||||
three_candidate_child_maps: None,
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A failed child guard cannot leak its parent update into restoration."""
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
child = await integration_session.get(ApiKey, key_hash)
|
||||
assert child is not None
|
||||
parent = ApiKey(hashed_key="failover-parent", balance=10_000_000)
|
||||
child.parent_key_hash = parent.hashed_key
|
||||
child.balance_limit = 75_000
|
||||
integration_session.add(parent)
|
||||
integration_session.add(child)
|
||||
await integration_session.commit()
|
||||
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
# The 100-sat candidate is rejected before forwarding; the third serves.
|
||||
assert [request.url.host for request in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"third.example.com",
|
||||
]
|
||||
|
||||
await integration_session.refresh(parent)
|
||||
await integration_session.refresh(child)
|
||||
assert parent.reserved_balance == 0
|
||||
assert child.reserved_balance == 0
|
||||
assert parent.total_spent == response.json()["cost"]["total_msats"]
|
||||
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert len(records) == 2
|
||||
assert sorted(record.status for record in records) == ["charged", "released"]
|
||||
assert len({record.reserved_msats for record in records}) == 1
|
||||
assert all(record.status != "active" for record in records)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def raised_envelope_provider_maps(
|
||||
patched_db_engine: None,
|
||||
@@ -504,6 +611,7 @@ async def raised_envelope_provider_maps(
|
||||
async def test_failover_reserves_serving_candidates_envelope(
|
||||
authenticated_client: AsyncClient,
|
||||
raised_envelope_provider_maps: None,
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""An affordable pricier fallback is re-reserved, served, and billed.
|
||||
|
||||
@@ -544,3 +652,15 @@ async def test_failover_reserves_serving_candidates_envelope(
|
||||
"expensive.example.com",
|
||||
]
|
||||
assert response.json()["cost"]["total_msats"] == 10_000
|
||||
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert len(records) == 2
|
||||
released = next(record for record in records if record.status == "released")
|
||||
charged = next(record for record in records if record.status == "charged")
|
||||
assert charged.reserved_msats > released.reserved_msats
|
||||
assert all(record.status != "active" for record in records)
|
||||
|
||||
@@ -38,7 +38,7 @@ async def test_overrun_charges_after_reservation_swept(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Overrun finalize must charge even when the reservation was already released."""
|
||||
from routstr.auth import adjust_payment_for_tokens
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
|
||||
deducted_max_cost = 990 # discounted reservation
|
||||
actual_token_cost = 1000 # actual cost overruns the reservation
|
||||
@@ -47,6 +47,10 @@ async def test_overrun_charges_after_reservation_swept(
|
||||
key = _make_key(balance=1000, reserved=0)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
key.reserved_balance = 0
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
@@ -79,8 +83,16 @@ async def test_free_response_path_closed_end_to_end(
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""A reservation released by the real sweeper must not yield a free response."""
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
from routstr.core.db import create_session, release_stale_reservations
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import (
|
||||
ReservationRelease,
|
||||
create_session,
|
||||
release_stale_reservations,
|
||||
)
|
||||
|
||||
deducted_max_cost = 990
|
||||
actual_token_cost = 1000
|
||||
@@ -104,10 +116,15 @@ async def test_free_response_path_closed_end_to_end(
|
||||
key = await session.get(ApiKey, key_hash)
|
||||
assert key is not None
|
||||
await pay_for_request(key, deducted_max_cost, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == deducted_max_cost
|
||||
key.reserved_at = int(time.time()) - 10_000
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert record is not None
|
||||
record.created_at = int(time.time()) - 10_000
|
||||
session.add(key)
|
||||
session.add(record)
|
||||
await session.commit()
|
||||
|
||||
# Sweeper releases the stale reservation without charging.
|
||||
@@ -129,18 +146,20 @@ async def test_free_response_path_closed_end_to_end(
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, session, deducted_max_cost, None, None
|
||||
key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
async with create_session() as session:
|
||||
final = await session.get(ApiKey, key_hash)
|
||||
assert final is not None
|
||||
|
||||
assert final.total_spent == actual_token_cost, (
|
||||
f"Free response: total_spent={final.total_spent}, expected {actual_token_cost}"
|
||||
)
|
||||
assert final.balance == 1000 - actual_token_cost, (
|
||||
f"Balance not charged after sweep: {final.balance}"
|
||||
)
|
||||
# Stale release is terminal for this reservation. A late finalizer must not
|
||||
# charge aggregate balance that may now belong to a newer request.
|
||||
assert final.total_spent == 0
|
||||
assert final.balance == 1000
|
||||
assert final.balance >= 0
|
||||
assert final.reserved_balance == 0
|
||||
|
||||
@@ -207,8 +207,35 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
|
||||
assert key.balance == model_cost # balance unchanged, only reserved goes up
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_model_maximum_is_required_and_reserved(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
from routstr.auth import pay_for_request, validate_bearer_key
|
||||
|
||||
short_key = _key(balance=95_000)
|
||||
exact_key = _key(balance=100_000)
|
||||
integration_session.add(short_key)
|
||||
integration_session.add(exact_key)
|
||||
await integration_session.commit()
|
||||
|
||||
with pytest.raises(HTTPException) as insufficient:
|
||||
await validate_bearer_key(
|
||||
f"sk-{short_key.hashed_key}", integration_session, min_cost=100_000
|
||||
)
|
||||
assert insufficient.value.status_code == 402
|
||||
|
||||
validated = await validate_bearer_key(
|
||||
f"sk-{exact_key.hashed_key}", integration_session, min_cost=100_000
|
||||
)
|
||||
await pay_for_request(validated, 100_000, integration_session)
|
||||
|
||||
await integration_session.refresh(exact_key)
|
||||
assert exact_key.reserved_balance == 100_000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6 — HTTP layer returns 402 JSON with the right shape
|
||||
# HTTP layer returns 402 JSON with the right shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -266,8 +293,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
|
||||
error = body["detail"]["error"]
|
||||
assert error["code"] == "insufficient_balance"
|
||||
assert error["type"] == "insufficient_quota"
|
||||
assert str(model_cost) in error["message"]
|
||||
assert str(user_balance) in error["message"]
|
||||
assert "622.888 sats (622888 msats) required" in error["message"]
|
||||
assert "20.32 sats (20320 msats) available" in error["message"]
|
||||
|
||||
# Balance must be completely untouched
|
||||
await integration_session.refresh(key)
|
||||
|
||||
@@ -3,20 +3,30 @@
|
||||
Covers two things:
|
||||
- The three constraint fields (balance_limit, balance_limit_reset, validity_date)
|
||||
are persisted on LightningInvoice and survive a DB round-trip.
|
||||
- create_api_key_from_invoice propagates those fields to the created ApiKey,
|
||||
so the constraints are actually enforced when the key is used.
|
||||
- The production-path API-key record helper propagates those fields to the
|
||||
created ApiKey, so the constraints are actually enforced when the key is used.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import create_api_key_from_invoice
|
||||
from routstr.lightning import _create_api_key_record
|
||||
|
||||
|
||||
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
|
||||
wallet.proofs = []
|
||||
wallet.keysets = {}
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
|
||||
def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
@@ -39,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
def mock_wallet_mint() -> object:
|
||||
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
|
||||
wallet = AsyncMock()
|
||||
wallet.mint = AsyncMock(return_value=[])
|
||||
wallet.proofs = []
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
mock_get_wallet.return_value = wallet
|
||||
yield mock_get_wallet
|
||||
|
||||
@@ -48,6 +66,7 @@ def mock_wallet_mint() -> object:
|
||||
# Persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_persists_balance_limit(
|
||||
integration_session: AsyncSession,
|
||||
@@ -92,6 +111,7 @@ async def test_invoice_persists_validity_date(
|
||||
# Propagation to ApiKey
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_created_key_receives_balance_limit(
|
||||
integration_session: AsyncSession,
|
||||
@@ -100,7 +120,7 @@ async def test_created_key_receives_balance_limit(
|
||||
integration_session.add(invoice)
|
||||
await integration_session.flush()
|
||||
|
||||
api_key = await create_api_key_from_invoice(invoice, integration_session)
|
||||
api_key = await _create_api_key_record(invoice, integration_session)
|
||||
await integration_session.commit()
|
||||
|
||||
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
|
||||
@@ -116,7 +136,7 @@ async def test_created_key_receives_balance_limit_reset(
|
||||
integration_session.add(invoice)
|
||||
await integration_session.flush()
|
||||
|
||||
api_key = await create_api_key_from_invoice(invoice, integration_session)
|
||||
api_key = await _create_api_key_record(invoice, integration_session)
|
||||
await integration_session.commit()
|
||||
|
||||
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
|
||||
@@ -133,7 +153,7 @@ async def test_created_key_receives_validity_date(
|
||||
integration_session.add(invoice)
|
||||
await integration_session.flush()
|
||||
|
||||
api_key = await create_api_key_from_invoice(invoice, integration_session)
|
||||
api_key = await _create_api_key_record(invoice, integration_session)
|
||||
await integration_session.commit()
|
||||
|
||||
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
|
||||
@@ -141,6 +161,254 @@ async def test_created_key_receives_validity_date(
|
||||
assert stored_key.validity_date == expiry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_payment_check_releases_connection_during_mint_quote(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(id="inv_slow_quote", status="pending", paid_at=None)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(invoice)
|
||||
await setup.commit()
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
|
||||
async def quote_status(*args: object, **kwargs: object) -> MagicMock:
|
||||
assert integration_engine.pool.checkedout() == 0 # type: ignore[attr-defined]
|
||||
return MagicMock(paid=False)
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.get_mint_quote = AsyncMock(side_effect=quote_status)
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await check_invoice_payment(stored, session)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(id="inv_concurrent", status="pending", paid_at=None)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(invoice)
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
mint_calls = 0
|
||||
|
||||
async def single_use_mint(*args: object, **kwargs: object) -> list[object]:
|
||||
# Real mints enforce single-use quotes: the second concurrent minter
|
||||
# gets rejected at the mint, mirroring cashu quote semantics.
|
||||
nonlocal mint_calls
|
||||
mint_calls += 1
|
||||
call_number = mint_calls
|
||||
await asyncio.sleep(0.05)
|
||||
if call_number > 1:
|
||||
raise Exception("quote already issued")
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=single_use_mint)
|
||||
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as first,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as second,
|
||||
):
|
||||
first_invoice = await first.get(LightningInvoice, invoice.id)
|
||||
second_invoice = await second.get(LightningInvoice, invoice.id)
|
||||
assert first_invoice is not None
|
||||
assert second_invoice is not None
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await asyncio.gather(
|
||||
check_invoice_payment(first_invoice, first),
|
||||
check_invoice_payment(second_invoice, second),
|
||||
)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored_invoice is not None
|
||||
assert stored_invoice.status == "paid"
|
||||
assert stored_invoice.api_key_hash is not None
|
||||
stored_key = await verify.get(ApiKey, stored_invoice.api_key_hash)
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == invoice.amount_sats * 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_mint_marks_invoice_for_settlement_retry(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(id="inv_mint_failure", status="pending", paid_at=None)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(invoice)
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await check_invoice_payment(stored, session)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "settlement_pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unpaid_topup_does_not_query_target_key(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(
|
||||
id="inv_unpaid_topup",
|
||||
status="pending",
|
||||
paid_at=None,
|
||||
purpose="topup",
|
||||
api_key_hash="target-key",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(invoice)
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=False))
|
||||
create_session = MagicMock(side_effect=RuntimeError("target lookup should not run"))
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", create_session),
|
||||
):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await check_invoice_payment(stored, session)
|
||||
|
||||
wallet.get_mint_quote.assert_awaited_once_with(invoice.payment_hash)
|
||||
create_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_topup_target_is_rejected_before_mint(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(
|
||||
id="inv_missing_topup_target",
|
||||
status="pending",
|
||||
paid_at=None,
|
||||
purpose="topup",
|
||||
api_key_hash="pruned-key",
|
||||
expires_at=int(time.time()) - 1,
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(invoice)
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock()
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.logger.critical") as critical,
|
||||
):
|
||||
from routstr.lightning import get_invoice_status
|
||||
|
||||
response = await get_invoice_status(invoice.id, session)
|
||||
|
||||
assert response.status == "reconciliation_required"
|
||||
assert stored.status == "reconciliation_required"
|
||||
assert stored not in session.dirty
|
||||
critical.assert_called_once()
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
async with AsyncSession(integration_engine) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "reconciliation_required"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _make_invoice(id="inv_finalize_failure", status="pending", paid_at=None)
|
||||
sibling = _make_invoice(
|
||||
id="inv_finalize_failure_sibling",
|
||||
bolt11="lnbc1000n1sibling",
|
||||
payment_hash="cafebabe" * 8,
|
||||
status="pending",
|
||||
paid_at=None,
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add_all([invoice, sibling])
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=successful_mint)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
stored_sibling = await session.get(LightningInvoice, sibling.id)
|
||||
assert stored is not None
|
||||
assert stored_sibling is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._create_api_key_record",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
),
|
||||
):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await check_invoice_payment(stored, session)
|
||||
|
||||
stored_state = inspect(stored)
|
||||
sibling_state = inspect(stored_sibling)
|
||||
assert stored_state is not None
|
||||
assert sibling_state is not None
|
||||
assert stored_state.expired is False
|
||||
assert sibling_state.expired is False
|
||||
assert stored.status == "settlement_pending"
|
||||
assert stored_sibling.id == sibling.id
|
||||
|
||||
assert wallet.mint.await_count == 1
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "settlement_pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_created_key_without_constraints_has_none_fields(
|
||||
integration_session: AsyncSession,
|
||||
@@ -149,7 +417,7 @@ async def test_created_key_without_constraints_has_none_fields(
|
||||
integration_session.add(invoice)
|
||||
await integration_session.flush()
|
||||
|
||||
api_key = await create_api_key_from_invoice(invoice, integration_session)
|
||||
api_key = await _create_api_key_record(invoice, integration_session)
|
||||
await integration_session.commit()
|
||||
|
||||
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
|
||||
@@ -157,3 +425,87 @@ async def test_created_key_without_constraints_has_none_fields(
|
||||
assert stored_key.balance_limit is None
|
||||
assert stored_key.balance_limit_reset is None
|
||||
assert stored_key.validity_date is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_db_guard_credits_once_when_both_mints_succeed(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""Even if the mint fails to enforce single-use quotes and both racers
|
||||
mint successfully, the conditional status update must credit exactly once."""
|
||||
key = ApiKey(hashed_key="race-key", balance=1_000)
|
||||
invoice = _make_invoice(
|
||||
id="inv_db_guard",
|
||||
status="pending",
|
||||
paid_at=None,
|
||||
purpose="topup",
|
||||
api_key_hash="race-key",
|
||||
)
|
||||
sibling = _make_invoice(
|
||||
id="inv_db_guard_sibling",
|
||||
bolt11="lnbc1000n1race-sibling",
|
||||
payment_hash="01234567" * 8,
|
||||
status="pending",
|
||||
paid_at=None,
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add_all([key, invoice, sibling])
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
await asyncio.sleep(0.05)
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
|
||||
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as first,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as second,
|
||||
):
|
||||
first_invoice = await first.get(LightningInvoice, invoice.id)
|
||||
first_sibling = await first.get(LightningInvoice, sibling.id)
|
||||
second_invoice = await second.get(LightningInvoice, invoice.id)
|
||||
assert first_invoice is not None
|
||||
assert first_sibling is not None
|
||||
assert second_invoice is not None
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
from routstr.lightning import check_invoice_payment
|
||||
|
||||
await asyncio.gather(
|
||||
check_invoice_payment(first_invoice, first),
|
||||
check_invoice_payment(second_invoice, second),
|
||||
)
|
||||
|
||||
first_state = inspect(first_invoice)
|
||||
sibling_state = inspect(first_sibling)
|
||||
second_state = inspect(second_invoice)
|
||||
assert first_state is not None
|
||||
assert sibling_state is not None
|
||||
assert second_state is not None
|
||||
assert first_state.expired is False
|
||||
assert sibling_state.expired is False
|
||||
assert second_state.expired is False
|
||||
assert first_invoice.id == invoice.id
|
||||
assert first_sibling.id == sibling.id
|
||||
assert second_invoice.id == invoice.id
|
||||
assert first_invoice.status == "paid"
|
||||
assert second_invoice.status == "paid"
|
||||
assert first_invoice not in first.dirty
|
||||
assert second_invoice not in second.dirty
|
||||
|
||||
assert wallet.mint.await_count == 1
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored_invoice is not None
|
||||
assert stored_invoice.status == "paid"
|
||||
stored_key = await verify.get(ApiKey, "race-key")
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 1_000 + invoice.amount_sats * 1000
|
||||
|
||||
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
|
||||
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
|
||||
counter = {"n": 0}
|
||||
|
||||
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
|
||||
async def fake_generate(
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
counter["n"] += 1
|
||||
return (
|
||||
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
|
||||
f"payment_hash_{counter['n']}",
|
||||
"http://localhost:3338",
|
||||
)
|
||||
|
||||
with patch(
|
||||
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
|
||||
body = resp.json()
|
||||
assert body["amount_sats"] == 500
|
||||
assert body["bolt11"].startswith("lnbc")
|
||||
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
|
||||
assert allowed_mints == ["http://localhost:3338"]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel import col, update
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import (
|
||||
_expire_invoice_if_authoritatively_unpaid,
|
||||
_finalize_invoice_settlement,
|
||||
_InvoiceSettlement,
|
||||
check_invoice_payment,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice(**overrides: object) -> LightningInvoice:
|
||||
suffix = uuid.uuid4().hex
|
||||
values = {
|
||||
"id": f"invoice-{suffix}",
|
||||
"bolt11": f"lnbc-{suffix}",
|
||||
"amount_sats": 100,
|
||||
"description": "settlement test",
|
||||
"payment_hash": f"quote-{suffix}",
|
||||
"status": "pending",
|
||||
"purpose": "create",
|
||||
"mint_url": "http://mint:3338",
|
||||
"expires_at": int(time.time()) + 3600,
|
||||
}
|
||||
values.update(overrides)
|
||||
return LightningInvoice(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_read_transaction_closes_before_external_mint_io(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice()
|
||||
integration_session.add(invoice)
|
||||
await integration_session.commit()
|
||||
stored = await integration_session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
|
||||
|
||||
async def get_wallet_without_open_db_transaction(
|
||||
*args: object, **kwargs: object
|
||||
) -> Mock:
|
||||
assert not integration_session.in_transaction()
|
||||
return wallet
|
||||
|
||||
with patch(
|
||||
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
|
||||
):
|
||||
await check_invoice_payment(stored, integration_session)
|
||||
|
||||
assert not integration_session.in_transaction()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_separate_sessions_cas_topup_credit_exactly_once(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
|
||||
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
|
||||
):
|
||||
results = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
|
||||
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
|
||||
)
|
||||
|
||||
assert sorted(settled for settled, _ in results) == [False, True]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_invoice is not None
|
||||
assert stored_invoice.status == "paid"
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
async def debit_balance(session: AsyncSession) -> None:
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key_hash)
|
||||
.values(balance=col(ApiKey.balance) - 10_000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
await session.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as debit,
|
||||
):
|
||||
settlement_result, _ = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
|
||||
debit_balance(debit),
|
||||
)
|
||||
|
||||
assert settlement_result[0]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 190_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
with patch.object(
|
||||
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
|
||||
):
|
||||
with pytest.raises(Exception, match="db unavailable"):
|
||||
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
settled, _ = await _finalize_invoice_settlement(
|
||||
snapshot, retry, 1_700_000_001
|
||||
)
|
||||
assert settled
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
wallet = Mock(
|
||||
proofs=[],
|
||||
keysets={"keyset-1": Mock()},
|
||||
load_proofs=AsyncMock(),
|
||||
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
)
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
stored = await failed.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(side_effect=Exception("db unavailable")),
|
||||
),
|
||||
):
|
||||
await check_invoice_payment(stored, failed)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "settlement_pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
stored = await retry.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
await check_invoice_payment(stored, retry)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expiry_cas_cannot_overwrite_concurrent_paid_invoice(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice(expires_at=0)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as caller:
|
||||
stale = await caller.get(LightningInvoice, invoice.id)
|
||||
assert stale is not None
|
||||
await caller.commit()
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as paid:
|
||||
result = await paid.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(col(LightningInvoice.id) == invoice.id)
|
||||
.values(status="paid", paid_at=123)
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
await paid.commit()
|
||||
|
||||
expired = await _expire_invoice_if_authoritatively_unpaid(
|
||||
stale, caller, True
|
||||
)
|
||||
|
||||
assert expired is False
|
||||
assert stale.status == "paid"
|
||||
assert stale.paid_at == 123
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "paid"
|
||||
assert stored.paid_at == 123
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_quote_worker_does_not_mint_after_expiry_claim_wins(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice(expires_at=0)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
quote_started = asyncio.Event()
|
||||
release_quote = asyncio.Event()
|
||||
|
||||
async def paid_quote_after_expiry(*_args: object, **_kwargs: object) -> Mock:
|
||||
quote_started.set()
|
||||
await release_quote.wait()
|
||||
return Mock(paid=True)
|
||||
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(side_effect=paid_quote_after_expiry),
|
||||
mint=AsyncMock(),
|
||||
)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as worker:
|
||||
observed_pending = await worker.get(LightningInvoice, invoice.id)
|
||||
assert observed_pending is not None
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
settlement_task = asyncio.create_task(
|
||||
check_invoice_payment(observed_pending, worker)
|
||||
)
|
||||
await quote_started.wait()
|
||||
|
||||
async with AsyncSession(
|
||||
integration_engine, expire_on_commit=False
|
||||
) as expirer:
|
||||
expiry_view = await expirer.get(LightningInvoice, invoice.id)
|
||||
assert expiry_view is not None
|
||||
await expirer.commit()
|
||||
assert await _expire_invoice_if_authoritatively_unpaid(
|
||||
expiry_view, expirer, True
|
||||
)
|
||||
|
||||
release_quote.set()
|
||||
assert await settlement_task is False
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "expired"
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Money-safety regression coverage for automatic wallet payouts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable, Coroutine
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core import db
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import credit_balance, periodic_payout
|
||||
|
||||
PRIMARY_MINT = "http://primary:3338"
|
||||
REFUND_MINT = "http://refund:3338"
|
||||
PAYOUT_INTERVAL = 987
|
||||
|
||||
|
||||
class _LoopBreak(Exception):
|
||||
"""Stop the otherwise-infinite payout loop after one cycle."""
|
||||
|
||||
|
||||
def _one_payout_cycle() -> Callable[[float], Coroutine[Any, Any, None]]:
|
||||
intervals_seen = 0
|
||||
|
||||
async def sleep(seconds: float) -> None:
|
||||
nonlocal intervals_seen
|
||||
if seconds == PAYOUT_INTERVAL:
|
||||
intervals_seen += 1
|
||||
if intervals_seen == 2:
|
||||
raise _LoopBreak()
|
||||
|
||||
return sleep
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_mint_liability_is_not_paid_as_owner_profit(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""Refund preferences must not make primary-mint customer funds payable."""
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(
|
||||
ApiKey(
|
||||
hashed_key="cross-mint-key",
|
||||
balance=50_000,
|
||||
refund_mint_url=REFUND_MINT,
|
||||
refund_currency="sat",
|
||||
)
|
||||
)
|
||||
await setup.commit()
|
||||
|
||||
primary_proof = MagicMock(amount=50)
|
||||
raw_send = AsyncMock(return_value=50)
|
||||
|
||||
def proofs_for_mint(
|
||||
_wallet: object, mint_url: str, unit: str, **_kwargs: object
|
||||
) -> list[MagicMock]:
|
||||
if mint_url == PRIMARY_MINT and unit == "sat":
|
||||
return [primary_proof]
|
||||
return []
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", [REFUND_MINT]),
|
||||
patch.object(settings, "primary_mint", PRIMARY_MINT),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.test"),
|
||||
patch.object(settings, "payout_interval_seconds", PAYOUT_INTERVAL),
|
||||
patch.object(settings, "min_payout_sat", 10),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_payout_cycle()),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(side_effect=proofs_for_mint),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, _wallet: proofs),
|
||||
),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
|
||||
):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
raw_send.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_payout_does_not_send_proofs_whose_liability_commit_is_in_flight(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""Proof visibility before liability commit must not expose customer funds."""
|
||||
key = ApiKey(
|
||||
hashed_key="in-flight-topup-key",
|
||||
balance=0,
|
||||
refund_mint_url=PRIMARY_MINT,
|
||||
refund_currency="sat",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
|
||||
setup.add(key)
|
||||
await setup.commit()
|
||||
|
||||
proofs: list[MagicMock] = []
|
||||
proof_visible = asyncio.Event()
|
||||
finish_redemption = asyncio.Event()
|
||||
liability_read = asyncio.Event()
|
||||
|
||||
async def redeem_token(
|
||||
token: str,
|
||||
destination_mint: str | None = None,
|
||||
destination_unit: str | None = None,
|
||||
) -> tuple[int, str, str]:
|
||||
proofs.append(MagicMock(amount=200))
|
||||
proof_visible.set()
|
||||
await finish_redemption.wait()
|
||||
return 200, "sat", PRIMARY_MINT
|
||||
|
||||
real_total_liability = db.total_user_liability
|
||||
|
||||
async def read_liability(_session: AsyncSession) -> int:
|
||||
async with db.create_session() as snapshot_session:
|
||||
value = await real_total_liability(snapshot_session)
|
||||
liability_read.set()
|
||||
return value
|
||||
|
||||
raw_send = AsyncMock(return_value=200)
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", PRIMARY_MINT),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.test"),
|
||||
patch.object(settings, "payout_interval_seconds", PAYOUT_INTERVAL),
|
||||
patch.object(settings, "min_payout_sat", 10),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_payout_cycle()),
|
||||
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=redeem_token)),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(side_effect=lambda *_args, **_kwargs: list(proofs)),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda visible, _wallet: visible),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.total_user_liability",
|
||||
AsyncMock(side_effect=read_liability),
|
||||
),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
|
||||
):
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as credit_session:
|
||||
stored_key = await credit_session.get(ApiKey, key.hashed_key)
|
||||
assert stored_key is not None
|
||||
credit_task = asyncio.create_task(
|
||||
credit_balance("cashu-token", stored_key, credit_session)
|
||||
)
|
||||
await asyncio.wait_for(proof_visible.wait(), timeout=2)
|
||||
|
||||
payout_task = asyncio.create_task(periodic_payout())
|
||||
try:
|
||||
await asyncio.wait_for(liability_read.wait(), timeout=0.1)
|
||||
liability_was_read_while_crediting = True
|
||||
except TimeoutError:
|
||||
liability_was_read_while_crediting = False
|
||||
|
||||
finish_redemption.set()
|
||||
await asyncio.wait_for(credit_task, timeout=2)
|
||||
|
||||
with pytest.raises(_LoopBreak):
|
||||
await asyncio.wait_for(payout_task, timeout=2)
|
||||
|
||||
assert liability_was_read_while_crediting is False
|
||||
raw_send.assert_not_awaited()
|
||||
@@ -0,0 +1,602 @@
|
||||
"""Real-database tests for the PPQ auto top-up claim lifecycle.
|
||||
|
||||
These exercise the claim against actual SQL rather than mocked sessions,
|
||||
because the guarantees under test are all about what the database will and
|
||||
will not let two concurrent writers do.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlmodel import select
|
||||
|
||||
from routstr.core.db import CashuTransaction, create_session
|
||||
from routstr.upstream.auto_topup import (
|
||||
PPQ_PHASE_CLAIMED,
|
||||
PPQ_PHASE_IN_FLIGHT,
|
||||
PPQ_PHASE_RECONCILE,
|
||||
_claim_ppq_topup,
|
||||
_ppq_payment_id,
|
||||
_ppq_payment_usd,
|
||||
_ppq_request_id,
|
||||
_ppq_spent_last_24h_usd,
|
||||
_ppq_state_id_for_provider,
|
||||
_record_ppq_invoice,
|
||||
_set_ppq_state_terminal,
|
||||
get_ppq_auto_topup_state,
|
||||
release_ppq_auto_topup_state,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
def _row(provider_id: int = 1) -> MagicMock:
|
||||
row = MagicMock()
|
||||
row.id = provider_id
|
||||
return row
|
||||
|
||||
|
||||
async def _seed_provider(provider_id: int = 1, slug: str = "ppq") -> None:
|
||||
"""Claim creation is fenced on the provider row existing; seed it."""
|
||||
from routstr.core.db import UpstreamProviderRow
|
||||
|
||||
async with create_session() as session:
|
||||
session.add(
|
||||
UpstreamProviderRow(
|
||||
id=provider_id,
|
||||
slug=slug,
|
||||
provider_type="ppqai",
|
||||
base_url="https://api.ppq.ai",
|
||||
api_key="secret",
|
||||
enabled=True,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _state_row(provider_id: int = 1) -> CashuTransaction | None:
|
||||
async with create_session() as session:
|
||||
return await session.get(
|
||||
CashuTransaction, _ppq_state_id_for_provider(provider_id)
|
||||
)
|
||||
|
||||
|
||||
async def _seed_claim(
|
||||
provider_id: int,
|
||||
phase: str,
|
||||
invoice_id: str,
|
||||
lease_expires_at: int,
|
||||
quote_id: str = "quote-1",
|
||||
) -> str:
|
||||
"""Seed a claim row and return its state token (the full request_id)."""
|
||||
token = _ppq_request_id(
|
||||
"operation-1", lease_expires_at, phase, invoice_id, quote_id
|
||||
)
|
||||
async with create_session() as session:
|
||||
session.add(
|
||||
CashuTransaction(
|
||||
id=_ppq_state_id_for_provider(provider_id),
|
||||
token="lnbc-invoice",
|
||||
amount=102,
|
||||
unit="sat",
|
||||
type="out",
|
||||
request_id=token,
|
||||
mint_url="https://mint.test",
|
||||
collected=False,
|
||||
source="ppq_auto_topup",
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return token
|
||||
|
||||
|
||||
async def test_second_claim_is_refused_while_the_first_is_active(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
await _seed_provider()
|
||||
assert await _claim_ppq_topup(_row()) is not None
|
||||
# The whole point of the claim: a concurrent cycle must not get one.
|
||||
assert await _claim_ppq_topup(_row()) is None
|
||||
|
||||
async with create_session() as session:
|
||||
rows = (await session.exec(select(CashuTransaction))).all()
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
async def test_claim_is_reusable_once_the_previous_attempt_finished(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
await _seed_provider()
|
||||
first = await _claim_ppq_topup(_row())
|
||||
assert first is not None
|
||||
assert await _set_ppq_state_terminal(_row(), first, collected=True, swept=False)
|
||||
|
||||
second = await _claim_ppq_topup(_row())
|
||||
assert second is not None and second != first
|
||||
|
||||
|
||||
async def test_recording_the_invoice_moves_the_claim_in_flight(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
await _seed_provider()
|
||||
operation_id = await _claim_ppq_topup(_row())
|
||||
assert operation_id is not None
|
||||
|
||||
state = await get_ppq_auto_topup_state(1)
|
||||
assert state["phase"] == PPQ_PHASE_CLAIMED
|
||||
assert state["releasable"] is True
|
||||
assert state["invoice_id"] is None
|
||||
|
||||
lease = await _record_ppq_invoice(
|
||||
_row(),
|
||||
operation_id,
|
||||
invoice="lnbc-invoice",
|
||||
invoice_id="invoice-1",
|
||||
quote_id="quote-1",
|
||||
amount=102,
|
||||
amount_usd=10,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
)
|
||||
assert lease > int(time.time())
|
||||
|
||||
state = await get_ppq_auto_topup_state(1)
|
||||
assert state["phase"] == PPQ_PHASE_IN_FLIGHT
|
||||
assert state["invoice_id"] == "invoice-1"
|
||||
# A payment is committed to a mint, so an admin must not sweep it.
|
||||
assert state["releasable"] is False
|
||||
# The raw BOLT11 invoice must never reach the admin API.
|
||||
assert "token" not in state
|
||||
|
||||
|
||||
async def test_release_refuses_an_in_flight_claim(patched_db_engine: Any) -> None:
|
||||
token = await _seed_claim(
|
||||
1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900
|
||||
)
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(1, state_token=token)
|
||||
|
||||
assert outcome.released is False
|
||||
assert outcome.reason == "payment_in_flight"
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is False
|
||||
|
||||
|
||||
async def test_release_refuses_a_stale_state_token(patched_db_engine: Any) -> None:
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(1, state_token="ppq:stale:token")
|
||||
|
||||
assert outcome.released is False
|
||||
assert outcome.reason == "stale_state"
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is False
|
||||
|
||||
|
||||
async def test_release_accepts_a_reconcile_claim(patched_db_engine: Any) -> None:
|
||||
token = await _seed_claim(
|
||||
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
|
||||
)
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(1, state_token=token)
|
||||
|
||||
assert outcome.released is True
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is True
|
||||
|
||||
|
||||
async def test_expired_in_flight_claim_becomes_releasable(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
# A worker that died mid-payment must not lock the provider forever.
|
||||
token = await _seed_claim(1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) - 1)
|
||||
|
||||
assert (await get_ppq_auto_topup_state(1))["releasable"] is True
|
||||
outcome = await release_ppq_auto_topup_state(1, state_token=token)
|
||||
assert outcome.released is True
|
||||
|
||||
|
||||
async def test_release_reports_no_active_claim_once_swept(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
token = await _seed_claim(
|
||||
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
|
||||
)
|
||||
assert (await release_ppq_auto_topup_state(1, state_token=token)).released
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(1, state_token=token)
|
||||
assert outcome.released is False
|
||||
assert outcome.reason == "no_active_claim"
|
||||
|
||||
|
||||
async def test_terminal_write_fails_after_the_claim_was_released(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
"""The symptom an admin release leaves behind for the owning worker."""
|
||||
token = await _seed_claim(
|
||||
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
|
||||
)
|
||||
assert (await release_ppq_auto_topup_state(1, state_token=token)).released
|
||||
|
||||
assert (
|
||||
await _set_ppq_state_terminal(
|
||||
_row(), "operation-1", collected=True, swept=False
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
async def test_ppq_claim_rows_are_excluded_from_the_admin_transaction_list(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.core.admin import get_transactions_api
|
||||
|
||||
await _seed_provider()
|
||||
await _claim_ppq_topup(_row())
|
||||
async with create_session() as session:
|
||||
session.add(
|
||||
CashuTransaction(
|
||||
id="real-transaction",
|
||||
token="cashuAreal",
|
||||
amount=50,
|
||||
unit="sat",
|
||||
type="out",
|
||||
source="x-cashu",
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
result = await get_transactions_api()
|
||||
|
||||
ids = {t["id"] for t in result["transactions"]} # type: ignore[index,union-attr]
|
||||
assert "real-transaction" in ids
|
||||
assert _ppq_state_id_for_provider(1) not in ids
|
||||
|
||||
|
||||
async def test_ppq_payment_audit_row_is_visible_and_survives_next_claim(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.core.admin import get_transactions_api
|
||||
|
||||
await _seed_provider()
|
||||
operation_id = await _claim_ppq_topup(_row())
|
||||
assert operation_id is not None
|
||||
await _record_ppq_invoice(
|
||||
_row(),
|
||||
operation_id,
|
||||
invoice="lnbc-secret-invoice",
|
||||
invoice_id="invoice-1",
|
||||
quote_id="quote-1",
|
||||
amount=102,
|
||||
amount_usd=10,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
)
|
||||
assert await _set_ppq_state_terminal(
|
||||
_row(), operation_id, collected=True, swept=False
|
||||
)
|
||||
|
||||
result = await get_transactions_api(source="ppq_auto_topup")
|
||||
transactions = result["transactions"]
|
||||
assert len(transactions) == 1
|
||||
audit = transactions[0]
|
||||
assert audit["id"] == _ppq_payment_id(operation_id)
|
||||
assert audit["token"] == "ppq-invoice:invoice-1:usd:10"
|
||||
assert audit["collected"] is True
|
||||
assert "lnbc-secret-invoice" not in audit["token"]
|
||||
|
||||
# Reusing the deterministic claim lock must not overwrite history.
|
||||
assert await _claim_ppq_topup(_row()) is not None
|
||||
async with create_session() as session:
|
||||
assert await session.get(CashuTransaction, audit["id"]) is not None
|
||||
|
||||
|
||||
async def test_reconcile_settles_a_recorded_invoice(patched_db_engine: Any) -> None:
|
||||
from routstr.upstream.auto_topup import _reconcile_ppq_state
|
||||
|
||||
await _seed_claim(1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900)
|
||||
provider = MagicMock()
|
||||
provider.check_topup_status = AsyncMock(return_value=True)
|
||||
|
||||
# Still suppresses this cycle, but the claim is now finished.
|
||||
assert await _reconcile_ppq_state(_row(), provider) is True
|
||||
|
||||
row = await _state_row()
|
||||
assert row is not None and row.collected is True
|
||||
|
||||
|
||||
async def test_stale_token_from_before_a_phase_change_cannot_release(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
"""The blocker scenario: admin reviews `claimed`, payment turns ambiguous.
|
||||
|
||||
The operation id is identical in both states, so an id-based fence would
|
||||
let the stale confirmation land. The full state token must not.
|
||||
"""
|
||||
await _seed_provider()
|
||||
operation_id = await _claim_ppq_topup(_row())
|
||||
assert operation_id is not None
|
||||
reviewed = await get_ppq_auto_topup_state(1)
|
||||
assert reviewed["phase"] == PPQ_PHASE_CLAIMED
|
||||
|
||||
# Worker records the invoice: same operation, new phase, proofs committed.
|
||||
await _record_ppq_invoice(
|
||||
_row(),
|
||||
operation_id,
|
||||
invoice="lnbc-invoice",
|
||||
invoice_id="invoice-1",
|
||||
quote_id="quote-1",
|
||||
amount=102,
|
||||
amount_usd=10,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
)
|
||||
|
||||
outcome = await release_ppq_auto_topup_state(
|
||||
1, state_token=str(reviewed["state_token"])
|
||||
)
|
||||
assert outcome.released is False
|
||||
assert outcome.reason == "stale_state"
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is False
|
||||
|
||||
|
||||
async def test_concurrent_claims_only_one_wins(patched_db_engine: Any) -> None:
|
||||
import asyncio
|
||||
|
||||
await _seed_provider()
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_claim_ppq_topup(_row()) for _ in range(5)), return_exceptions=True
|
||||
)
|
||||
winners = [r for r in results if isinstance(r, str)]
|
||||
assert len(winners) == 1
|
||||
|
||||
async with create_session() as session:
|
||||
rows = (await session.exec(select(CashuTransaction))).all()
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
async def test_reconcile_releases_claim_when_mint_reports_unpaid(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.upstream.auto_topup import _reconcile_ppq_state
|
||||
|
||||
# Lease expired, PPQ never credited: only the mint's own "unpaid" answer
|
||||
# may hand the claim back.
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
|
||||
provider = MagicMock()
|
||||
provider.check_topup_status = AsyncMock(return_value=False)
|
||||
|
||||
with patch(
|
||||
"routstr.upstream.auto_topup.check_bolt11_payment_status",
|
||||
AsyncMock(return_value="unpaid"),
|
||||
) as status:
|
||||
suppressed = await _reconcile_ppq_state(_row(), provider)
|
||||
|
||||
status.assert_awaited_once_with("https://mint.test", "sat", "quote-1")
|
||||
assert suppressed is False
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is True
|
||||
|
||||
|
||||
async def test_reconcile_keeps_claim_when_mint_answer_is_not_final(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.upstream.auto_topup import _reconcile_ppq_state
|
||||
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
|
||||
provider = MagicMock()
|
||||
provider.check_topup_status = AsyncMock(return_value=False)
|
||||
|
||||
for answer in ("paid", "pending", "unknown"):
|
||||
with patch(
|
||||
"routstr.upstream.auto_topup.check_bolt11_payment_status",
|
||||
AsyncMock(return_value=answer),
|
||||
):
|
||||
assert await _reconcile_ppq_state(_row(), provider) is True
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is False, answer
|
||||
|
||||
|
||||
async def test_release_endpoint_maps_refusals_to_409(patched_db_engine: Any) -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.core.admin import (
|
||||
ReleasePPQAutoTopupRequest,
|
||||
release_ppq_auto_topup_api,
|
||||
)
|
||||
|
||||
provider_row = MagicMock()
|
||||
provider_row.provider_type = "ppqai"
|
||||
|
||||
token = await _seed_claim(
|
||||
1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900
|
||||
)
|
||||
|
||||
with patch(
|
||||
"routstr.core.admin._require_ppq_provider",
|
||||
AsyncMock(return_value=provider_row),
|
||||
):
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await release_ppq_auto_topup_api(
|
||||
1,
|
||||
ReleasePPQAutoTopupRequest(
|
||||
confirmed_safe_to_retry=True, state_token=token
|
||||
),
|
||||
)
|
||||
assert excinfo.value.status_code == 409
|
||||
assert "in flight" in excinfo.value.detail
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await release_ppq_auto_topup_api(
|
||||
1,
|
||||
ReleasePPQAutoTopupRequest(
|
||||
confirmed_safe_to_retry=True, state_token="ppq:wrong"
|
||||
),
|
||||
)
|
||||
assert excinfo.value.status_code == 409
|
||||
assert "changed since" in excinfo.value.detail
|
||||
|
||||
|
||||
async def test_provider_delete_is_blocked_by_an_active_claim(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.core.admin import delete_upstream_provider
|
||||
from routstr.core.db import UpstreamProviderRow
|
||||
|
||||
async with create_session() as session:
|
||||
session.add(
|
||||
UpstreamProviderRow(
|
||||
id=1,
|
||||
slug="ppq",
|
||||
provider_type="ppqai",
|
||||
base_url="https://api.ppq.ai",
|
||||
api_key="secret",
|
||||
enabled=True,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await delete_upstream_provider("1")
|
||||
assert excinfo.value.status_code == 409
|
||||
|
||||
# Provider must still exist.
|
||||
async with create_session() as session:
|
||||
assert await session.get(UpstreamProviderRow, 1) is not None
|
||||
|
||||
|
||||
async def test_claim_is_refused_when_the_provider_row_is_gone(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
"""The worker's half of the delete race: no provider row, no claim."""
|
||||
assert await _claim_ppq_topup(_row()) is None
|
||||
|
||||
async with create_session() as session:
|
||||
rows = (await session.exec(select(CashuTransaction))).all()
|
||||
assert rows == []
|
||||
|
||||
|
||||
async def test_claim_is_refused_after_a_provider_type_change(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.core.db import UpstreamProviderRow
|
||||
|
||||
await _seed_provider()
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, 1)
|
||||
assert provider is not None
|
||||
provider.provider_type = "openai"
|
||||
session.add(provider)
|
||||
await session.commit()
|
||||
|
||||
assert await _claim_ppq_topup(_row()) is None
|
||||
|
||||
|
||||
async def test_disabled_provider_with_claim_still_reconciles(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
"""A claim tracks committed money; eligibility must not stop reconciling."""
|
||||
from routstr.core.db import UpstreamProviderRow
|
||||
from routstr.upstream.auto_topup import _reconcile_all_ppq_claims
|
||||
|
||||
await _seed_provider()
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, 1)
|
||||
assert provider is not None
|
||||
provider.enabled = False
|
||||
session.add(provider)
|
||||
await session.commit()
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
|
||||
|
||||
ppq = MagicMock()
|
||||
ppq.check_topup_status = AsyncMock(return_value=True)
|
||||
with patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=ppq,
|
||||
):
|
||||
await _reconcile_all_ppq_claims()
|
||||
|
||||
row = await _state_row()
|
||||
assert row is not None and row.collected is True
|
||||
|
||||
|
||||
async def test_claim_without_api_key_still_reconciles_via_the_mint(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
from routstr.core.db import UpstreamProviderRow
|
||||
from routstr.upstream.auto_topup import _reconcile_all_ppq_claims
|
||||
|
||||
await _seed_provider()
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, 1)
|
||||
assert provider is not None
|
||||
provider.api_key = ""
|
||||
session.add(provider)
|
||||
await session.commit()
|
||||
# Lease expired, so the mint may be consulted.
|
||||
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
|
||||
|
||||
with patch(
|
||||
"routstr.upstream.auto_topup.check_bolt11_payment_status",
|
||||
AsyncMock(return_value="unpaid"),
|
||||
) as status:
|
||||
await _reconcile_all_ppq_claims()
|
||||
|
||||
# No API key: PPQ was never polled, but the mint was, and its definitive
|
||||
# "unpaid" released the claim.
|
||||
status.assert_awaited_once()
|
||||
row = await _state_row()
|
||||
assert row is not None and row.swept is True
|
||||
|
||||
|
||||
def test_ppq_payment_usd_prefers_stamped_amount() -> None:
|
||||
# Stamped rows must not move with the BTC price.
|
||||
assert _ppq_payment_usd(102, "sat", "ppq-invoice:a:usd:10", 0.5) == 10.0
|
||||
|
||||
|
||||
def test_ppq_payment_usd_falls_back_to_current_price() -> None:
|
||||
# Rows recorded before the stamp existed convert sats at today's price.
|
||||
assert _ppq_payment_usd(2000, "sat", "ppq-invoice:legacy", 0.001) == 2.0
|
||||
assert _ppq_payment_usd(2_000_000, "msat", "ppq-invoice:legacy", 0.001) == 2.0
|
||||
|
||||
|
||||
def test_ppq_payment_usd_survives_malformed_stamp() -> None:
|
||||
assert _ppq_payment_usd(3000, "sat", "ppq-invoice:x:usd:oops", 0.001) == 3.0
|
||||
|
||||
|
||||
async def test_daily_spend_ignores_provably_unattempted_payments(
|
||||
patched_db_engine: Any,
|
||||
) -> None:
|
||||
def _payment(
|
||||
id_: str, token: str, collected: bool, swept: bool
|
||||
) -> CashuTransaction:
|
||||
return CashuTransaction(
|
||||
id=id_,
|
||||
token=token,
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
source="ppq_auto_topup",
|
||||
collected=collected,
|
||||
swept=swept,
|
||||
)
|
||||
|
||||
async with create_session() as session:
|
||||
# Settled, in-flight, and provably-unattempted payments plus a
|
||||
# pre-stamp row: only the unattempted one must be excluded.
|
||||
session.add(_payment("pay-usd-1", "ppq-invoice:a:usd:100", True, False))
|
||||
session.add(_payment("pay-usd-2", "ppq-invoice:b:usd:50", False, False))
|
||||
session.add(_payment("pay-usd-3", "ppq-invoice:c:usd:25", False, True))
|
||||
legacy = _payment("pay-usd-4", "ppq-invoice:legacy", True, False)
|
||||
legacy.amount = 2000
|
||||
session.add(legacy)
|
||||
await session.commit()
|
||||
|
||||
assert await _ppq_spent_last_24h_usd(0.001) == 152.0
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Integration coverage for proxy database-session lifetime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import Response
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr import proxy as proxy_module
|
||||
from routstr.core.db import ApiKey
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_proxy_releases_db_connection_before_upstream_headers(
|
||||
integration_engine: AsyncEngine,
|
||||
integration_session: AsyncSession,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""Slow upstream header waits must not retain a checked-out DB connection."""
|
||||
key = ApiKey(
|
||||
hashed_key="proxy-pool-key",
|
||||
balance=1_000_000,
|
||||
refund_mint_url="http://primary:3338",
|
||||
refund_currency="sat",
|
||||
)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
|
||||
request = MagicMock()
|
||||
request.method = "POST"
|
||||
request.headers = {"authorization": "Bearer test-key"}
|
||||
request.body = AsyncMock(return_value=json.dumps({"model": "test-model"}).encode())
|
||||
request.url.path = "/v1/chat/completions"
|
||||
request.state.request_id = "pool-hold-regression"
|
||||
|
||||
model = MagicMock()
|
||||
upstream = MagicMock()
|
||||
upstream.provider_type = "test"
|
||||
upstream.prepare_headers.return_value = {}
|
||||
|
||||
async def wait_for_headers(*args: object, **kwargs: object) -> Response:
|
||||
assert integration_engine.pool.checkedout() == 0 # type: ignore[attr-defined]
|
||||
return Response(status_code=200)
|
||||
|
||||
upstream.forward_request = AsyncMock(side_effect=wait_for_headers)
|
||||
|
||||
with (
|
||||
patch("routstr.proxy.get_candidates", return_value=[(model, upstream)]),
|
||||
patch("routstr.proxy.get_max_cost_for_model", AsyncMock(return_value=100)),
|
||||
patch(
|
||||
"routstr.proxy.calculate_discounted_max_cost",
|
||||
AsyncMock(return_value=100),
|
||||
),
|
||||
patch("routstr.proxy.check_token_balance"),
|
||||
patch("routstr.proxy.get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
):
|
||||
response = await proxy_module._proxy(
|
||||
request, "v1/chat/completions", integration_session
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -126,8 +126,11 @@ async def test_parent_and_child_keys_are_not_pruned(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
|
||||
"""A key referenced by a pending topup invoice is never pruned mid-topup."""
|
||||
@pytest.mark.parametrize("status", ["pending", "settlement_pending"])
|
||||
async def test_retryable_invoice_protects_key(
|
||||
patched_db_engine: None, status: str
|
||||
) -> None:
|
||||
"""A key referenced by a retryable topup invoice is never pruned mid-topup."""
|
||||
key = _dead_key(LONG_AGO)
|
||||
invoice = LightningInvoice(
|
||||
id=f"inv_{uuid.uuid4().hex}",
|
||||
@@ -135,7 +138,7 @@ async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
|
||||
amount_sats=10,
|
||||
description="topup",
|
||||
payment_hash=uuid.uuid4().hex,
|
||||
status="pending",
|
||||
status=status,
|
||||
api_key_hash=key.hashed_key,
|
||||
purpose="topup",
|
||||
expires_at=NOW + 10_000,
|
||||
|
||||
@@ -141,7 +141,7 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
Previously this would drive reserved_balance negative. With the floor guard,
|
||||
it should return False and leave reserved_balance at 0.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_key_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
@@ -151,8 +151,12 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 100, integration_session)
|
||||
test_key.reserved_balance = 0
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
|
||||
# Try to revert more than available — should be a no-op
|
||||
# A stale cleanup already released the aggregate reservation.
|
||||
result = await revert_pay_for_request(test_key, integration_session, 100)
|
||||
|
||||
await integration_session.refresh(test_key)
|
||||
@@ -161,8 +165,8 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
assert test_key.reserved_balance == 0, (
|
||||
f"Reserved balance should remain 0, got: {test_key.reserved_balance}"
|
||||
)
|
||||
assert test_key.total_requests == 0, (
|
||||
f"Total requests should remain 0, got: {test_key.total_requests}"
|
||||
assert test_key.total_requests == 1, (
|
||||
f"Total requests should remain 1, got: {test_key.total_requests}"
|
||||
)
|
||||
|
||||
|
||||
@@ -171,17 +175,18 @@ async def test_revert_with_sufficient_reserved_balance_succeeds(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that revert_pay_for_request works correctly when there is enough reserved balance."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_ok_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=5000,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
|
||||
result = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
|
||||
@@ -202,17 +207,21 @@ async def test_revert_partial_reserved_balance_is_noop(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that reverting more than the current reserved_balance is a no-op."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_partial_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=5000,
|
||||
reserved_balance=50,
|
||||
total_requests=1,
|
||||
reserved_balance=0,
|
||||
total_requests=0,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
test_key.reserved_balance = 50
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
|
||||
# Try to revert 500 when only 50 is reserved — should be no-op
|
||||
result = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
@@ -237,20 +246,28 @@ async def test_double_revert_prevented(
|
||||
This simulates the double-revert scenario where both upstream/base.py
|
||||
and proxy.py attempt to revert the same reservation.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
unique_key = f"test_double_revert_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=5,
|
||||
reserved_balance=0,
|
||||
total_requests=4,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(test_key, integration_session)
|
||||
|
||||
# First revert — should succeed
|
||||
result1 = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
result1 = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(test_key)
|
||||
|
||||
assert result1 is True
|
||||
@@ -258,7 +275,9 @@ async def test_double_revert_prevented(
|
||||
assert test_key.total_requests == 4
|
||||
|
||||
# Second revert of the same amount — should be no-op
|
||||
result2 = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
result2 = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(test_key)
|
||||
|
||||
assert result2 is False, "Second revert should be a no-op"
|
||||
@@ -279,22 +298,30 @@ async def test_sequential_reverts_never_go_negative(
|
||||
Simulates the double-revert scenario where multiple code paths
|
||||
attempt to revert the same reservation.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
unique_key = f"test_multi_revert_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=5,
|
||||
reserved_balance=0,
|
||||
total_requests=4,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(test_key, integration_session)
|
||||
|
||||
# Run 5 sequential reverts for the same 500 reservation
|
||||
results = []
|
||||
for _ in range(5):
|
||||
r = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
r = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
results.append(r)
|
||||
|
||||
await integration_session.refresh(test_key)
|
||||
@@ -317,7 +344,11 @@ async def test_child_key_revert_floor_guard(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that child key reserved_balance also has floor guard on revert."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
parent_key_hash = f"test_parent_{uuid.uuid4().hex[:8]}"
|
||||
child_key_hash = f"test_child_{uuid.uuid4().hex[:8]}"
|
||||
@@ -325,22 +356,26 @@ async def test_child_key_revert_floor_guard(
|
||||
parent_key = ApiKey(
|
||||
hashed_key=parent_key_hash,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
)
|
||||
child_key = ApiKey(
|
||||
hashed_key=child_key_hash,
|
||||
balance=0,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
parent_key_hash=parent_key_hash,
|
||||
)
|
||||
integration_session.add(parent_key)
|
||||
integration_session.add(child_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(child_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(child_key, integration_session)
|
||||
|
||||
# First revert succeeds
|
||||
result1 = await revert_pay_for_request(child_key, integration_session, 500)
|
||||
result1 = await revert_pay_for_request(
|
||||
child_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(parent_key)
|
||||
await integration_session.refresh(child_key)
|
||||
|
||||
@@ -349,7 +384,9 @@ async def test_child_key_revert_floor_guard(
|
||||
assert child_key.reserved_balance == 0
|
||||
|
||||
# Second revert is a no-op for both parent and child
|
||||
result2 = await revert_pay_for_request(child_key, integration_session, 500)
|
||||
result2 = await revert_pay_for_request(
|
||||
child_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(parent_key)
|
||||
await integration_session.refresh(child_key)
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from collections.abc import Callable
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from routstr.core.settings import settings
|
||||
@@ -28,7 +29,9 @@ from routstr.core.settings import settings
|
||||
# with the testmint stub that bypasses swapping (see conftest.py).
|
||||
from routstr.wallet import recieve_token as _real_recieve_token
|
||||
|
||||
PRIMARY_MINT = "http://primary:3338"
|
||||
# Match the authenticated fixture's persisted refund mint: existing-key topups
|
||||
# are intentionally constrained to that mint for collateral provenance.
|
||||
PRIMARY_MINT = "http://localhost:3338"
|
||||
|
||||
|
||||
def _make_swap_mocks(
|
||||
@@ -81,7 +84,9 @@ def _make_swap_mocks(
|
||||
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
|
||||
)
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(return_value=Mock())
|
||||
mock_token_wallet.melt = AsyncMock(
|
||||
return_value=Mock(state=MeltQuoteState.paid)
|
||||
)
|
||||
|
||||
return mock_token, mock_token_wallet, mock_primary_wallet
|
||||
|
||||
@@ -89,7 +94,12 @@ def _make_swap_mocks(
|
||||
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
|
||||
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
|
||||
|
||||
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
|
||||
def fake_get_wallet(
|
||||
mint_url: str,
|
||||
unit: str = "sat",
|
||||
load: bool = True,
|
||||
**kwargs: object,
|
||||
) -> Mock:
|
||||
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
|
||||
|
||||
return fake_get_wallet
|
||||
@@ -139,7 +149,7 @@ async def test_topup_retries_when_melt_demands_more_than_quoted(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 179, needed: 180 (Code: 11000)"
|
||||
),
|
||||
Mock(),
|
||||
Mock(state=MeltQuoteState.paid),
|
||||
]
|
||||
|
||||
response = await _post_topup(
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Restart reconciliation for ambiguous melts, against a real cashu wallet DB.
|
||||
|
||||
The ambiguous-melt path in ``execute_bolt11_payment`` re-reserves proofs with
|
||||
``set_reserved_for_melt(..., quote_id=...)`` after cashu's ``melt()`` clears
|
||||
both the reservation and the ``melt_id`` on a transport error. These tests
|
||||
prove, on cashu's actual sqlite store rather than mocks, that the recovery
|
||||
survives a process restart: a fresh wallet instance on the same database can
|
||||
still find the proofs by ``melt_id`` — the lookup ``get_melt_quote()`` uses to
|
||||
invalidate them on "paid" or release them on "unpaid".
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from cashu.wallet import crud
|
||||
from cashu.wallet.wallet import Wallet
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
QUOTE_ID = "quote-restart-1"
|
||||
|
||||
|
||||
def _proof(secret: str, amount: int = 64) -> Proof:
|
||||
return Proof(
|
||||
id="009a1f293253e41e",
|
||||
amount=amount,
|
||||
secret=secret,
|
||||
C="02bc9097997d81afb2cc7346b5e4345a9346bd2a506eb7958598a72f0cf85163ea",
|
||||
)
|
||||
|
||||
|
||||
async def _wallet(db_dir: Path) -> Wallet:
|
||||
# with_db builds the instance and runs migrations locally; nothing here
|
||||
# talks to a mint.
|
||||
return await Wallet.with_db("https://mint.test", str(db_dir))
|
||||
|
||||
|
||||
async def _seed_ambiguous_melt(wallet: Wallet) -> list[Proof]:
|
||||
"""Reproduce the exact sequence of an ambiguous melt failure.
|
||||
|
||||
1. Proofs exist and are selected for a melt.
|
||||
2. cashu's melt() reserves them with the quote id, then hits a transport
|
||||
error and rolls that back — reservation gone, melt_id gone.
|
||||
3. Our recovery in execute_bolt11_payment re-reserves with the quote id.
|
||||
"""
|
||||
proofs = [_proof("secret-a"), _proof("secret-b", amount=32)]
|
||||
for proof in proofs:
|
||||
await crud.store_proof(proof, db=wallet.db)
|
||||
|
||||
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
|
||||
# cashu's `except` block in melt():
|
||||
await wallet.set_reserved_for_melt(proofs, reserved=False, quote_id=None)
|
||||
# our recovery:
|
||||
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
|
||||
return proofs
|
||||
|
||||
|
||||
async def test_melt_recovery_is_findable_by_quote_after_restart(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
wallet = await _wallet(tmp_path)
|
||||
await _seed_ambiguous_melt(wallet)
|
||||
|
||||
# "Restart": a brand-new wallet on the same database file, as after a
|
||||
# process crash between the melt and any reconciliation.
|
||||
restarted = await _wallet(tmp_path)
|
||||
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
|
||||
|
||||
# This is get_melt_quote()'s own lookup. If it comes back empty, a "paid"
|
||||
# answer can never invalidate these proofs and an "unpaid" answer can
|
||||
# never release them — the strand the send-style re-reserve caused.
|
||||
assert sorted(p.secret for p in found) == ["secret-a", "secret-b"]
|
||||
assert all(p.reserved for p in found)
|
||||
assert all(p.melt_id == QUOTE_ID for p in found)
|
||||
|
||||
|
||||
async def test_send_style_reservation_would_not_be_reconcilable(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The defect the fix removed, demonstrated on the real store."""
|
||||
wallet = await _wallet(tmp_path)
|
||||
proofs = [_proof("secret-send")]
|
||||
for proof in proofs:
|
||||
await crud.store_proof(proof, db=wallet.db)
|
||||
|
||||
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
|
||||
await wallet.set_reserved_for_melt(proofs, reserved=False, quote_id=None)
|
||||
# The old recovery: reserve as a send, no quote association.
|
||||
await wallet.set_reserved_for_send(proofs, reserved=True)
|
||||
|
||||
restarted = await _wallet(tmp_path)
|
||||
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
|
||||
assert found == [] # reconciliation would never see these proofs
|
||||
|
||||
|
||||
async def test_unpaid_reconciliation_releases_recovered_proofs_after_restart(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The full recovery arc: crash, restart, mint says unpaid, funds usable."""
|
||||
wallet = await _wallet(tmp_path)
|
||||
await _seed_ambiguous_melt(wallet)
|
||||
|
||||
restarted = await _wallet(tmp_path)
|
||||
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
|
||||
assert len(found) == 2
|
||||
|
||||
# What get_melt_quote() does on an "unpaid" answer.
|
||||
await restarted.set_reserved_for_melt(found, reserved=False, quote_id=None)
|
||||
|
||||
released = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
|
||||
assert released == []
|
||||
all_proofs = await crud.get_proofs(db=restarted.db)
|
||||
assert len(all_proofs) == 2
|
||||
assert all(not p.reserved for p in all_proofs) # spendable again
|
||||
@@ -0,0 +1,35 @@
|
||||
from contextlib import asynccontextmanager
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core.admin import get_transactions_api
|
||||
from routstr.core.db import CashuTransaction
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transactions_api_excludes_internal_sweep_claim_timestamp() -> None:
|
||||
transaction = CashuTransaction(
|
||||
token="cashu-token",
|
||||
amount=10,
|
||||
unit="sat",
|
||||
type="out",
|
||||
sweep_started_at=123,
|
||||
)
|
||||
count_result = MagicMock()
|
||||
count_result.one.return_value = 1
|
||||
transactions_result = MagicMock()
|
||||
transactions_result.all.return_value = [transaction]
|
||||
session = MagicMock()
|
||||
session.exec = AsyncMock(side_effect=[count_result, transactions_result])
|
||||
|
||||
@asynccontextmanager
|
||||
async def create_session(): # type: ignore[no-untyped-def]
|
||||
yield session
|
||||
|
||||
with patch("routstr.core.admin.create_session", create_session):
|
||||
response = await get_transactions_api()
|
||||
|
||||
assert response["total"] == 1
|
||||
assert response["transactions"][0]["token"] == "cashu-token"
|
||||
assert "sweep_started_at" not in response["transactions"][0]
|
||||
@@ -1,8 +1,12 @@
|
||||
import base64
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
import routstr.wallet as wallet_module
|
||||
from routstr.core import admin
|
||||
|
||||
|
||||
@@ -13,20 +17,12 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
|
||||
) -> None:
|
||||
primary_mint = "https://primary.example"
|
||||
effective_mint = requested_mint or primary_mint
|
||||
wallet = object()
|
||||
proofs = [SimpleNamespace(amount=40), SimpleNamespace(amount=60)]
|
||||
token = "cashuBoutgoing"
|
||||
|
||||
get_wallet = AsyncMock(return_value=wallet)
|
||||
get_proofs = Mock(return_value=proofs)
|
||||
filter_proofs = AsyncMock(return_value=proofs)
|
||||
send_token = AsyncMock(return_value=token)
|
||||
store_transaction = AsyncMock(return_value=True)
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", get_wallet)
|
||||
monkeypatch.setattr(admin, "get_proofs_per_mint_and_unit", get_proofs)
|
||||
monkeypatch.setattr(admin, "slow_filter_spend_proofs", filter_proofs)
|
||||
monkeypatch.setattr(admin, "send_token", send_token)
|
||||
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=effective_mint))
|
||||
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
|
||||
monkeypatch.setattr(admin.settings, "primary_mint", primary_mint)
|
||||
|
||||
@@ -35,10 +31,7 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
|
||||
admin.WithdrawRequest(amount=75, mint_url=requested_mint, unit="sat"),
|
||||
)
|
||||
|
||||
assert result == {"token": token}
|
||||
get_wallet.assert_awaited_once_with(effective_mint, "sat")
|
||||
get_proofs.assert_called_once_with(wallet, effective_mint, "sat", not_reserved=True)
|
||||
filter_proofs.assert_awaited_once_with(proofs, wallet)
|
||||
assert result == {"token": token, "mint_url": effective_mint}
|
||||
send_token.assert_awaited_once_with(75, "sat", effective_mint)
|
||||
store_transaction.assert_awaited_once_with(
|
||||
token=token,
|
||||
@@ -56,17 +49,10 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mint = "https://primary.example"
|
||||
proofs = [SimpleNamespace(amount=100)]
|
||||
token = "cashuBrecoverable"
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", AsyncMock(return_value=object()))
|
||||
monkeypatch.setattr(
|
||||
admin, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
admin, "slow_filter_spend_proofs", AsyncMock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(admin, "send_token", AsyncMock(return_value=token))
|
||||
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=mint))
|
||||
monkeypatch.setattr(
|
||||
admin,
|
||||
"store_cashu_transaction",
|
||||
@@ -78,5 +64,89 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
|
||||
|
||||
result = await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
|
||||
|
||||
assert result == {"token": token}
|
||||
assert result == {"token": token, "mint_url": mint}
|
||||
critical.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_falls_back_from_insufficient_preferred_mint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
requested_mint = "https://primary.example"
|
||||
actual_mint = "https://secondary.example"
|
||||
proofs = [SimpleNamespace(amount=100, reserved=False, id="00")]
|
||||
token_payload = {
|
||||
"token": [
|
||||
{
|
||||
"mint": actual_mint,
|
||||
"proofs": [
|
||||
{
|
||||
"id": "00",
|
||||
"amount": 75,
|
||||
"secret": "secret",
|
||||
"C": "02" + "00" * 32,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"unit": "sat",
|
||||
}
|
||||
token = "cashuA" + base64.urlsafe_b64encode(
|
||||
json.dumps(token_payload).encode()
|
||||
).decode()
|
||||
wallet = SimpleNamespace(
|
||||
keysets={},
|
||||
proofs=proofs,
|
||||
select_to_send=AsyncMock(return_value=(proofs, 0)),
|
||||
serialize_proofs=AsyncMock(return_value=token),
|
||||
set_reserved_for_send=AsyncMock(),
|
||||
)
|
||||
find_funded = AsyncMock(return_value=actual_mint)
|
||||
store_transaction = AsyncMock(return_value=True)
|
||||
|
||||
monkeypatch.setattr(wallet_module, "find_trusted_mint_with_funds", find_funded)
|
||||
monkeypatch.setattr(wallet_module, "get_wallet", AsyncMock(return_value=wallet))
|
||||
monkeypatch.setattr(
|
||||
wallet_module, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
|
||||
|
||||
result = await admin.withdraw(
|
||||
Mock(), admin.WithdrawRequest(amount=75, mint_url=requested_mint)
|
||||
)
|
||||
|
||||
assert result == {"token": token, "mint_url": actual_mint}
|
||||
find_funded.assert_awaited_once_with(
|
||||
75, "sat", requested_mint, force_reload=True
|
||||
)
|
||||
wallet.select_to_send.assert_awaited_once()
|
||||
store_transaction.assert_awaited_once_with(
|
||||
token=token,
|
||||
amount=75,
|
||||
unit="sat",
|
||||
mint_url=actual_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_maps_true_aggregate_insufficient_funds_to_400(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
admin,
|
||||
"send_token",
|
||||
AsyncMock(
|
||||
side_effect=ValueError(
|
||||
"No trusted mint has 75 sat available; balances={'mint': 0}"
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "Insufficient wallet balance"
|
||||
|
||||
@@ -270,6 +270,54 @@ async def test_internal_error_with_invalid_keyword_does_not_masquerade(
|
||||
assert await session.get(ApiKey, hashed_key) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_msat_token_sets_provenance_without_cashu_mint_duplicate(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
token = "cashuAprimary_msat_token"
|
||||
token_obj = SimpleNamespace(mint="http://primary:3338", unit="msat")
|
||||
credit = AsyncMock(return_value=1_000)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", token_obj.mint),
|
||||
patch.object(settings, "primary_mint_unit", "msat"),
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
|
||||
patch("routstr.auth.credit_balance", new=credit),
|
||||
):
|
||||
key = await validate_bearer_key(token, session)
|
||||
|
||||
assert key.refund_mint_url == token_obj.mint
|
||||
assert key.refund_currency == "msat"
|
||||
credit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_token_unit_mismatch_is_rejected_before_redemption(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
token = "cashuAprimary_wrong_unit"
|
||||
token_obj = SimpleNamespace(mint="http://primary:3338", unit="sat")
|
||||
credit = AsyncMock(return_value=1_000)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", token_obj.mint),
|
||||
patch.object(settings, "primary_mint_unit", "msat"),
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
|
||||
patch("routstr.auth.credit_balance", new=credit),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await validate_bearer_key(token, session)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
credit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_cashu_token_returns_400_invalid_token(
|
||||
session: AsyncSession,
|
||||
|
||||
@@ -4,7 +4,29 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from routstr.core.db import CashuTransaction
|
||||
from routstr.upstream.auto_topup import _check_and_topup
|
||||
from routstr.upstream.auto_topup import (
|
||||
_check_and_topup,
|
||||
_parse_ppq_request_id,
|
||||
_run_auto_topup_cycle,
|
||||
validate_ppq_auto_topup_settings,
|
||||
)
|
||||
from routstr.upstream.ppqai import PPQAIUpstreamProvider
|
||||
from routstr.wallet import Bolt11PaymentAmbiguous, Bolt11PaymentNotAttempted
|
||||
|
||||
|
||||
def test_ppq_claim_parser_rejects_invalid_expiry() -> None:
|
||||
assert (
|
||||
_parse_ppq_request_id("ppq:operation:not-a-timestamp:claimed:invoice:none")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_balance_rejects_boolean_api_value() -> None:
|
||||
provider = PPQAIUpstreamProvider("secret")
|
||||
provider.check_balance = AsyncMock(return_value={"balance": False}) # type: ignore[method-assign]
|
||||
|
||||
assert await provider.get_balance() is None
|
||||
|
||||
|
||||
def _row() -> MagicMock:
|
||||
@@ -12,6 +34,7 @@ def _row() -> MagicMock:
|
||||
row.id = "provider-1"
|
||||
row.base_url = "https://provider.test"
|
||||
row.api_key = "secret"
|
||||
row.provider_type = "routstr"
|
||||
row.provider_settings = json.dumps(
|
||||
{
|
||||
"auto_topup": True,
|
||||
@@ -66,6 +89,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(return_value=True),
|
||||
) as store,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.token_mint_url",
|
||||
return_value="https://fallback-mint.test",
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.create_session", return_value=session),
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
@@ -74,7 +101,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
token="cashu-token",
|
||||
amount=50,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
mint_url="https://fallback-mint.test",
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
@@ -138,6 +165,572 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.release_token_reservation",
|
||||
AsyncMock(),
|
||||
) as reclaim,
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
|
||||
reclaim.assert_awaited_once_with("cashu-token")
|
||||
provider.topup.assert_not_awaited()
|
||||
|
||||
|
||||
def _ppq_row() -> MagicMock:
|
||||
row = MagicMock()
|
||||
row.id = "ppq-provider-1"
|
||||
row.base_url = "https://api.ppq.ai"
|
||||
row.api_key = "secret"
|
||||
row.provider_type = "ppqai"
|
||||
row.provider_settings = json.dumps(
|
||||
{
|
||||
"auto_topup": True,
|
||||
"topup_threshold": 5.0,
|
||||
"topup_amount_limit": 10,
|
||||
}
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_auto_topup_pays_invoice_and_confirms_settlement() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
invoice_id="invoice-1",
|
||||
payment_request="lnbc-invoice",
|
||||
amount=10,
|
||||
currency="USD",
|
||||
expires_at=None,
|
||||
)
|
||||
)
|
||||
provider.check_topup_status = AsyncMock(return_value=True)
|
||||
plan = MagicMock()
|
||||
plan.invoice_amount_sats = 100
|
||||
plan.maximum_spend_sats = 102
|
||||
plan.quote.amount = 100
|
||||
plan.quote.fee_reserve = 2
|
||||
plan.mint_url = "https://mint-rich.test"
|
||||
plan.unit = "sat"
|
||||
row = _ppq_row()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(return_value="operation-1"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=0.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.prepare_bolt11_payment",
|
||||
AsyncMock(return_value=plan),
|
||||
) as prepare,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.execute_bolt11_payment",
|
||||
AsyncMock(return_value=(101, "https://mint-rich.test", "sat")),
|
||||
) as execute,
|
||||
patch("routstr.upstream.auto_topup._record_ppq_invoice", AsyncMock()) as record,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()
|
||||
) as record_spent,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
|
||||
) as terminal,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
):
|
||||
await _check_and_topup(row)
|
||||
|
||||
provider.initiate_topup.assert_awaited_once_with(10)
|
||||
prepare.assert_awaited_once_with("lnbc-invoice")
|
||||
execute.assert_awaited_once_with(plan)
|
||||
record.assert_awaited_once()
|
||||
record_spent.assert_awaited_once_with("operation-1", 101)
|
||||
provider.check_topup_status.assert_awaited_once_with("invoice-1")
|
||||
terminal.assert_awaited_once_with(row, "operation-1", collected=True, swept=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_ambiguous_melt_keeps_claim_and_emits_critical_alert() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
invoice_id="invoice-1",
|
||||
payment_request="lnbc-invoice",
|
||||
amount=10,
|
||||
currency="USD",
|
||||
expires_at=None,
|
||||
)
|
||||
)
|
||||
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
|
||||
plan.quote.amount = 100
|
||||
plan.quote.fee_reserve = 2
|
||||
row = _ppq_row()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(return_value="operation-1"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=0.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.prepare_bolt11_payment",
|
||||
AsyncMock(return_value=plan),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.execute_bolt11_payment",
|
||||
AsyncMock(side_effect=Bolt11PaymentAmbiguous("ambiguous melt")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._record_ppq_invoice",
|
||||
AsyncMock(return_value=2_000_000_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
|
||||
) as reconcile_mark,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
|
||||
) as terminal,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
patch("routstr.upstream.auto_topup.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(Bolt11PaymentAmbiguous, match="ambiguous melt"):
|
||||
await _check_and_topup(row)
|
||||
|
||||
# The claim is never released — it moves to reconcile for the admin.
|
||||
terminal.assert_not_awaited()
|
||||
reconcile_mark.assert_awaited_once()
|
||||
critical.assert_called_once()
|
||||
assert "admin reconciliation" in critical.call_args.args[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_payment_not_attempted_releases_claim_for_retry() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
invoice_id="invoice-1",
|
||||
payment_request="lnbc-invoice",
|
||||
amount=10,
|
||||
currency="USD",
|
||||
expires_at=None,
|
||||
)
|
||||
)
|
||||
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
|
||||
plan.quote.amount = 100
|
||||
plan.quote.fee_reserve = 2
|
||||
plan.quote.quote = "quote-1"
|
||||
terminal = AsyncMock(return_value=True)
|
||||
row = _ppq_row()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=0.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(return_value="operation-1"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.prepare_bolt11_payment",
|
||||
AsyncMock(return_value=plan),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._record_ppq_invoice",
|
||||
AsyncMock(return_value=2_000_000_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.execute_bolt11_payment",
|
||||
AsyncMock(side_effect=Bolt11PaymentNotAttempted("unpaid")),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup._set_ppq_state_terminal", terminal),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
|
||||
) as reconcile,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
pytest.raises(Bolt11PaymentNotAttempted, match="unpaid"),
|
||||
):
|
||||
await _check_and_topup(row)
|
||||
|
||||
terminal.assert_awaited_once_with(row, "operation-1", collected=False, swept=True)
|
||||
reconcile.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_status_error_after_payment_marks_reconcile_and_alerts() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
invoice_id="invoice-1",
|
||||
payment_request="lnbc-invoice",
|
||||
amount=10,
|
||||
currency="USD",
|
||||
expires_at=None,
|
||||
)
|
||||
)
|
||||
provider.check_topup_status = AsyncMock(side_effect=RuntimeError("PPQ 502"))
|
||||
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
|
||||
plan.quote.amount = 100
|
||||
plan.quote.fee_reserve = 2
|
||||
plan.quote.quote = "quote-1"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=0.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(return_value="operation-1"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.prepare_bolt11_payment",
|
||||
AsyncMock(return_value=plan),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._record_ppq_invoice",
|
||||
AsyncMock(return_value=2_000_000_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.execute_bolt11_payment",
|
||||
AsyncMock(return_value=(101, "https://mint.test", "sat")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()
|
||||
) as spent,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
|
||||
) as reconcile,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
|
||||
) as terminal,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
patch("routstr.upstream.auto_topup.logger.critical") as critical,
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
spent.assert_awaited_once_with("operation-1", 101)
|
||||
reconcile.assert_awaited_once()
|
||||
terminal.assert_not_awaited()
|
||||
assert "settlement polling failed" in critical.call_args.args[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_preflight_funding_check_happens_before_invoice_creation() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=1),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup._claim_ppq_topup", AsyncMock()) as claim,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
provider.initiate_topup.assert_not_awaited()
|
||||
claim.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_claim_at_cycle_start_suppresses_topup_for_whole_cycle() -> None:
|
||||
row = _ppq_row()
|
||||
row.id = 1
|
||||
session = AsyncMock()
|
||||
result = MagicMock()
|
||||
result.all.return_value = [row]
|
||||
session.exec.return_value = result
|
||||
context = MagicMock()
|
||||
context.__aenter__ = AsyncMock(return_value=session)
|
||||
context.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_all_ppq_claims",
|
||||
AsyncMock(return_value={1}),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.create_session", return_value=context),
|
||||
patch("routstr.upstream.auto_topup._check_and_topup", AsyncMock()) as check,
|
||||
):
|
||||
await _run_auto_topup_cycle()
|
||||
|
||||
check.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_auto_topup_skips_when_balance_meets_threshold() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=5.0)
|
||||
provider.initiate_topup = AsyncMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
provider.initiate_topup.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_auto_topup_skips_when_daily_spend_cap_reached() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000_000),
|
||||
),
|
||||
# 1000 USD already spent, exactly the daily cap: the next 10 USD
|
||||
# top-up must be refused.
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=1000.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(),
|
||||
) as claim,
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
claim.assert_not_awaited()
|
||||
provider.initiate_topup.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_pending_attempt_suppresses_duplicate_topup() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=True),
|
||||
),
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
provider.get_balance.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_auto_topup_rejects_non_finite_balance() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=float("nan"))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup._claim_ppq_topup", AsyncMock()) as claim,
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
claim.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settled_topup_alerts_when_its_claim_was_already_released() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=2.5)
|
||||
provider.initiate_topup = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
invoice_id="invoice-1",
|
||||
payment_request="lnbc-invoice",
|
||||
amount=10,
|
||||
currency="USD",
|
||||
expires_at=None,
|
||||
)
|
||||
)
|
||||
provider.check_topup_status = AsyncMock(return_value=True)
|
||||
plan = MagicMock()
|
||||
plan.maximum_spend_sats = 102
|
||||
plan.quote.amount = 100
|
||||
plan.quote.fee_reserve = 2
|
||||
plan.mint_url = "https://mint-rich.test"
|
||||
plan.unit = "sat"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._reconcile_ppq_state",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._claim_ppq_topup",
|
||||
AsyncMock(return_value="operation-1"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.prepare_bolt11_payment",
|
||||
AsyncMock(return_value=plan),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
|
||||
AsyncMock(return_value=10_000),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
|
||||
AsyncMock(return_value=0.0),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.execute_bolt11_payment",
|
||||
AsyncMock(return_value=(101, "https://mint-rich.test", "sat")),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup._record_ppq_invoice", AsyncMock()),
|
||||
patch("routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup._set_ppq_state_terminal",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
|
||||
patch("routstr.upstream.auto_topup.logger") as log,
|
||||
):
|
||||
await _check_and_topup(_ppq_row())
|
||||
|
||||
assert any(
|
||||
"claim was already released" in call.args[0]
|
||||
for call in log.critical.call_args_list
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("settings", "expected"),
|
||||
[
|
||||
({"auto_topup": False, "topup_threshold": -1}, None),
|
||||
(
|
||||
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 10},
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"auto_topup": True, "topup_threshold": None, "topup_amount_limit": 10},
|
||||
"threshold",
|
||||
),
|
||||
(
|
||||
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 0.5},
|
||||
"whole number",
|
||||
),
|
||||
(
|
||||
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 5000},
|
||||
"between",
|
||||
),
|
||||
(
|
||||
{"auto_topup": True, "topup_threshold": True, "topup_amount_limit": 10},
|
||||
"threshold",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ppq_auto_topup_settings_validation(
|
||||
settings: dict, expected: str | None
|
||||
) -> None:
|
||||
problem = validate_ppq_auto_topup_settings(settings)
|
||||
if expected is None:
|
||||
assert problem is None
|
||||
else:
|
||||
assert problem is not None and expected in problem
|
||||
|
||||
|
||||
def test_ppq_auto_topup_settings_validation_survives_huge_json_integers() -> None:
|
||||
# json.loads happily produces integers past float range; float() raises
|
||||
# OverflowError there instead of returning inf.
|
||||
problem = validate_ppq_auto_topup_settings(
|
||||
{"auto_topup": True, "topup_threshold": 10**400, "topup_amount_limit": 10}
|
||||
)
|
||||
assert problem is not None and "threshold" in problem
|
||||
|
||||
@@ -221,6 +221,78 @@ def _make_api_key(
|
||||
return key
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_returns_persisted_token_after_cache_loss() -> None:
|
||||
key = _make_api_key(balance=0, refund_currency="sat")
|
||||
refund_token = "cashuApersisted_refund_token"
|
||||
refund_tx = _make_cashu_tx(
|
||||
token=refund_token,
|
||||
amount=5,
|
||||
unit="sat",
|
||||
type="out",
|
||||
request_id=None,
|
||||
)
|
||||
refund_tx.source = "apikey"
|
||||
refund_tx.api_key_hashed_key = key.hashed_key
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=_exec_result(refund_tx))
|
||||
session.add = MagicMock()
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
|
||||
patch("routstr.balance.send_token", AsyncMock()) as mock_send_token,
|
||||
):
|
||||
result = await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert result == {"token": refund_token, "sats": "5"}
|
||||
assert refund_tx.collected is True
|
||||
session.add.assert_called_once_with(refund_tx)
|
||||
session.commit.assert_awaited_once()
|
||||
mock_send_token.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_rejects_persisted_token_after_sweep() -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
key = _make_api_key(balance=0, refund_currency="sat")
|
||||
refund_tx = _make_cashu_tx(
|
||||
token="cashuAswept_apikey_refund",
|
||||
amount=5,
|
||||
unit="sat",
|
||||
request_id=None,
|
||||
swept=True,
|
||||
)
|
||||
refund_tx.source = "apikey"
|
||||
refund_tx.api_key_hashed_key = key.hashed_key
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=_exec_result(refund_tx))
|
||||
session.add = MagicMock()
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 410
|
||||
assert exc_info.value.detail == "Refund has been swept"
|
||||
session.add.assert_not_called()
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_stores_cashu_transaction_with_apikey_source() -> None:
|
||||
key = _make_api_key(balance=5000, refund_currency="sat")
|
||||
@@ -534,6 +606,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
|
||||
assert exc_info.value.detail == "Cashu mint is unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.wallet import SourceMintConnectionError
|
||||
|
||||
key = _make_api_key(balance=1000)
|
||||
session = MagicMock()
|
||||
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
|
||||
|
||||
with (
|
||||
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
|
||||
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await topup_wallet_endpoint(
|
||||
cashu_token="cashuAtoken", key=key, session=session
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "cannot be redeemed at another mint" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_already_spent_still_returns_400() -> None:
|
||||
"""Regression: the mint-unreachable short-circuit must not swallow the
|
||||
@@ -686,3 +781,68 @@ async def test_topup_unexpected_non_valueerror_returns_500() -> None:
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert exc_info.value.detail == "Internal server error"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_ambiguous_melt_does_not_restore_balance() -> None:
|
||||
"""An ambiguous LNURL melt may still settle: the debit must be kept."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.payment.lnurl import MeltOutcomeAmbiguousError
|
||||
|
||||
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
|
||||
patch("routstr.balance._refund_cache_set", AsyncMock()),
|
||||
patch(
|
||||
"routstr.balance.send_to_lnurl",
|
||||
AsyncMock(side_effect=MeltOutcomeAmbiguousError("outcome is ambiguous")),
|
||||
),
|
||||
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 502
|
||||
mock_restore.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_clean_failure_still_restores_balance() -> None:
|
||||
"""A definitively failed melt must keep restoring the debited balance."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
|
||||
patch("routstr.balance._refund_cache_set", AsyncMock()),
|
||||
patch(
|
||||
"routstr.balance.send_to_lnurl",
|
||||
AsyncMock(side_effect=RuntimeError("mint rejected melt")),
|
||||
),
|
||||
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
mock_restore.assert_awaited_once()
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from routstr import balance as balance_module
|
||||
from routstr.core.db import get_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_balance_accepts_large_cashu_token_in_post_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
token = "cashuA" + "x" * 20_000
|
||||
key = SimpleNamespace(hashed_key="hashed", balance=123_000)
|
||||
validate_bearer_key = AsyncMock(return_value=key)
|
||||
session = AsyncMock()
|
||||
monkeypatch.setattr(balance_module, "validate_bearer_key", validate_bearer_key)
|
||||
|
||||
async def override_get_session(): # type: ignore[no-untyped-def]
|
||||
yield session
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(balance_module.balance_router)
|
||||
app.dependency_overrides[get_session] = override_get_session
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/v1/balance/create",
|
||||
json={"initial_balance_token": token},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"api_key": "sk-hashed", "balance": 123_000}
|
||||
validate_bearer_key.assert_awaited_once_with(token, session)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Real-DB coverage for db.balances_by_mint_and_unit.
|
||||
|
||||
Verifies the grouped liability query used by fetch_all_balances: it sums
|
||||
balances per (mint_url, unit), filters to the requested mints/units, excludes
|
||||
NULL mint/currency rows, and returns nothing for empty inputs.
|
||||
"""
|
||||
|
||||
from typing import AsyncGenerator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import (
|
||||
ApiKey,
|
||||
balance_for_mint_and_unit,
|
||||
balances_by_mint_and_unit,
|
||||
)
|
||||
|
||||
|
||||
def _make_engine() -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
"sqlite+aiosqlite://",
|
||||
poolclass=StaticPool,
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def session() -> "AsyncGenerator[AsyncSession, None]":
|
||||
engine = _make_engine()
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
db_session = AsyncSession(engine, expire_on_commit=False)
|
||||
try:
|
||||
yield db_session
|
||||
finally:
|
||||
await db_session.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _add_key(
|
||||
session: AsyncSession,
|
||||
hashed_key: str,
|
||||
balance: int,
|
||||
mint_url: str | None,
|
||||
currency: str | None,
|
||||
) -> None:
|
||||
session.add(
|
||||
ApiKey(
|
||||
hashed_key=hashed_key,
|
||||
balance=balance,
|
||||
refund_mint_url=mint_url,
|
||||
refund_currency=currency,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sums_and_groups_by_mint_and_unit(session: AsyncSession) -> None:
|
||||
await _add_key(session, "a", 1000, "http://m1", "sat")
|
||||
await _add_key(session, "b", 500, "http://m1", "sat")
|
||||
await _add_key(session, "c", 7000, "http://m1", "msat")
|
||||
await _add_key(session, "d", 200, "http://m2", "sat")
|
||||
|
||||
result = await balances_by_mint_and_unit(
|
||||
session, ["http://m1", "http://m2"], ["sat", "msat"]
|
||||
)
|
||||
|
||||
assert result[("http://m1", "sat")] == 1500
|
||||
assert result[("http://m1", "msat")] == 7000
|
||||
assert result[("http://m2", "sat")] == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filters_out_unrequested_mints_and_units(session: AsyncSession) -> None:
|
||||
await _add_key(session, "a", 1000, "http://wanted", "sat")
|
||||
await _add_key(session, "b", 999, "http://other", "sat")
|
||||
await _add_key(session, "c", 888, "http://wanted", "usd")
|
||||
|
||||
result = await balances_by_mint_and_unit(session, ["http://wanted"], ["sat"])
|
||||
|
||||
assert result == {("http://wanted", "sat"): 1000}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_excludes_rows_with_null_mint_or_currency(session: AsyncSession) -> None:
|
||||
await _add_key(session, "a", 1000, "http://m1", "sat")
|
||||
await _add_key(session, "b", 4242, None, None)
|
||||
|
||||
result = await balances_by_mint_and_unit(session, ["http://m1"], ["sat"])
|
||||
|
||||
assert result == {("http://m1", "sat"): 1000}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scalar_balance_for_one_mint_and_unit(session: AsyncSession) -> None:
|
||||
await _add_key(session, "a", 1000, "http://m1", "sat")
|
||||
await _add_key(session, "b", 500, "http://m1", "sat")
|
||||
await _add_key(session, "c", 9000, "http://m1", "msat")
|
||||
await _add_key(session, "d", 700, "http://m2", "sat")
|
||||
|
||||
assert await balance_for_mint_and_unit(session, "http://m1", "sat") == 1500
|
||||
assert await balance_for_mint_and_unit(session, "http://missing", "sat") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_inputs_return_empty_mapping(session: AsyncSession) -> None:
|
||||
await _add_key(session, "a", 1000, "http://m1", "sat")
|
||||
|
||||
assert await balances_by_mint_and_unit(session, [], ["sat"]) == {}
|
||||
assert await balances_by_mint_and_unit(session, ["http://m1"], []) == {}
|
||||
@@ -15,6 +15,7 @@ os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
|
||||
|
||||
from routstr.core.settings import settings
|
||||
from routstr.payment.cost_calculation import CostData, MaxCostData, calculate_cost
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -527,11 +528,136 @@ async def test_openrouter_upstream_inference_cost_components_are_used() -> None:
|
||||
result = await calculate_cost(response, max_cost=100000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert result.input_msats == 994
|
||||
assert result.output_msats == 3477
|
||||
assert result.input_msats == 995
|
||||
assert result.output_msats == 3476
|
||||
assert result.cache_read_msats == 758
|
||||
assert result.cache_creation_msats == 0
|
||||
assert result.input_msats + result.output_msats == result.total_msats == 4471
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usd_cache_breakdown_matches_token_priced_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Authoritative USD totals must retain model-specific cache-rate ratios."""
|
||||
monkeypatch.setattr(settings, "fixed_pricing", False)
|
||||
model = Model(
|
||||
id="cache-priced-model",
|
||||
name="cache-priced-model",
|
||||
created=0,
|
||||
description="",
|
||||
context_length=8192,
|
||||
architecture=Architecture(
|
||||
modality="text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="test",
|
||||
instruct_type=None,
|
||||
),
|
||||
pricing=Pricing(prompt=0.01, completion=0.02),
|
||||
sats_pricing=Pricing(
|
||||
prompt=0.01,
|
||||
completion=0.02,
|
||||
input_cache_read=0.001,
|
||||
input_cache_write=0.01,
|
||||
),
|
||||
per_request_limits=None,
|
||||
top_provider=None,
|
||||
)
|
||||
usage = {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 100,
|
||||
"prompt_tokens_details": {"cached_tokens": 900},
|
||||
}
|
||||
|
||||
token_result = await calculate_cost(
|
||||
{"model": model.id, "usage": usage},
|
||||
max_cost=100_000,
|
||||
model_obj=model,
|
||||
)
|
||||
usd_result = await calculate_cost(
|
||||
{
|
||||
"model": model.id,
|
||||
"usage": {
|
||||
**usage,
|
||||
"cost": 0.000195,
|
||||
"cost_details": {
|
||||
"input_cost": 0.000095,
|
||||
"output_cost": 0.0001,
|
||||
},
|
||||
},
|
||||
},
|
||||
max_cost=100_000,
|
||||
model_obj=model,
|
||||
provider_fee=1.0,
|
||||
)
|
||||
|
||||
assert isinstance(token_result, CostData)
|
||||
assert isinstance(usd_result, CostData)
|
||||
assert usd_result.total_msats == token_result.total_msats == 3900
|
||||
assert usd_result.input_msats + usd_result.output_msats == usd_result.total_msats
|
||||
assert usd_result.cache_read_msats == token_result.cache_read_msats == 900
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usd_cache_breakdown_does_not_absorb_total_rounding_remainder(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Sub-msat cache components truncate like the token-priced path."""
|
||||
monkeypatch.setattr(settings, "fixed_pricing", False)
|
||||
model = Model(
|
||||
id="sub-msat-cache-model",
|
||||
name="sub-msat-cache-model",
|
||||
created=0,
|
||||
description="",
|
||||
context_length=8192,
|
||||
architecture=Architecture(
|
||||
modality="text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="test",
|
||||
instruct_type=None,
|
||||
),
|
||||
pricing=Pricing(prompt=0.001, completion=0.001),
|
||||
sats_pricing=Pricing(
|
||||
prompt=0.001,
|
||||
completion=0.001,
|
||||
input_cache_write=0.0006,
|
||||
),
|
||||
per_request_limits=None,
|
||||
top_provider=None,
|
||||
)
|
||||
usage = {
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"cache_creation_input_tokens": 1,
|
||||
}
|
||||
|
||||
token_result = await calculate_cost(
|
||||
{"model": model.id, "usage": usage},
|
||||
max_cost=100_000,
|
||||
model_obj=model,
|
||||
)
|
||||
usd_result = await calculate_cost(
|
||||
{
|
||||
"model": model.id,
|
||||
"usage": {
|
||||
**usage,
|
||||
"cost": 0.00000003,
|
||||
"cost_details": {"input_cost": 0.00000003},
|
||||
},
|
||||
},
|
||||
max_cost=100_000,
|
||||
model_obj=model,
|
||||
provider_fee=1.0,
|
||||
)
|
||||
|
||||
assert isinstance(token_result, CostData)
|
||||
assert isinstance(usd_result, CostData)
|
||||
assert usd_result.total_msats == token_result.total_msats == 1
|
||||
assert usd_result.cache_creation_msats == token_result.cache_creation_msats == 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PPQ.AI BYOK: upstream_inference_cost + BYOK fee billing
|
||||
#
|
||||
@@ -568,12 +694,14 @@ async def test_ppq_byok_bills_upstream_inference_cost_plus_fee() -> None:
|
||||
# msats), not the fee alone (~0.0023 USD → ~45k msats). ~20× correction.
|
||||
assert result.total_msats == 940274
|
||||
assert result.input_msats + result.output_msats == result.total_msats
|
||||
assert result.input_msats == 926546
|
||||
assert result.output_msats == 13728
|
||||
assert result.input_msats == 926547
|
||||
assert result.output_msats == 13727
|
||||
assert result.total_usd == pytest.approx(0.047013667305)
|
||||
# Token normalisation (OpenAI dialect: cached included in prompt_tokens)
|
||||
assert result.input_tokens == 5070 # 164371 - 159301
|
||||
assert result.cache_read_input_tokens == 159301
|
||||
assert result.cache_read_msats == 897966
|
||||
assert result.cache_creation_msats == 0
|
||||
assert result.output_tokens == 99
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Response-contract tests for Routstr cost metadata across paid paths."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
|
||||
from routstr.core.db import ApiKey # noqa: E402
|
||||
from routstr.upstream.base import BaseUpstreamProvider # noqa: E402
|
||||
|
||||
COST_DATA = {
|
||||
"base_msats": 0,
|
||||
"input_msats": 1_200,
|
||||
"output_msats": 300,
|
||||
"total_msats": 1_500,
|
||||
"total_usd": 0.0001,
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 3,
|
||||
"cache_read_input_tokens": 8,
|
||||
"cache_creation_input_tokens": 2,
|
||||
"cache_read_msats": 80,
|
||||
"cache_creation_msats": 40,
|
||||
}
|
||||
|
||||
|
||||
def _provider() -> BaseUpstreamProvider:
|
||||
return BaseUpstreamProvider(base_url="http://test", api_key="upstream-key")
|
||||
|
||||
|
||||
def _key() -> ApiKey:
|
||||
return ApiKey(hashed_key="abcdef0123" * 4, balance=1_000_000)
|
||||
|
||||
|
||||
def _session() -> Any:
|
||||
session = MagicMock()
|
||||
session.refresh = AsyncMock()
|
||||
return session
|
||||
|
||||
|
||||
def _upstream_response(payload: dict) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=payload,
|
||||
request=httpx.Request("POST", "http://test"),
|
||||
)
|
||||
|
||||
|
||||
def _assert_cost_contract(response: Any) -> None:
|
||||
body = json.loads(response.body)
|
||||
assert body["usage"]["cost"] == {
|
||||
"base_msats": 0,
|
||||
"input_msats": 1_200,
|
||||
"output_msats": 300,
|
||||
"total_msats": 1_500,
|
||||
"total_usd": 0.0001,
|
||||
"cache_read_input_tokens": 8,
|
||||
"cache_creation_input_tokens": 2,
|
||||
"cache_read_msats": 80,
|
||||
"cache_creation_msats": 40,
|
||||
}
|
||||
assert response.headers["X-Routstr-Cost-Msats"] == "1500"
|
||||
assert response.headers["X-Routstr-Input-Cost-Msats"] == "1200"
|
||||
assert response.headers["X-Routstr-Output-Cost-Msats"] == "300"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_balance_chat_completion_uses_shared_cost_contract() -> None:
|
||||
provider = _provider()
|
||||
with patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
new=AsyncMock(return_value=dict(COST_DATA)),
|
||||
):
|
||||
response = await provider.handle_non_streaming_chat_completion(
|
||||
_upstream_response(
|
||||
{
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 3},
|
||||
}
|
||||
),
|
||||
_key(),
|
||||
_session(),
|
||||
deducted_max_cost=10_000,
|
||||
)
|
||||
|
||||
_assert_cost_contract(response)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_balance_responses_completion_uses_shared_cost_contract() -> None:
|
||||
provider = _provider()
|
||||
with patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
new=AsyncMock(return_value=dict(COST_DATA)),
|
||||
):
|
||||
response = await provider.handle_non_streaming_responses_completion(
|
||||
_upstream_response(
|
||||
{
|
||||
"model": "test-model",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 3},
|
||||
}
|
||||
),
|
||||
_key(),
|
||||
_session(),
|
||||
deducted_max_cost=10_000,
|
||||
)
|
||||
|
||||
_assert_cost_contract(response)
|
||||
@@ -4,7 +4,7 @@ Tests admin endpoints that are testable without full app setup:
|
||||
withdraw validation, authentication guards, and slug validation.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
@@ -46,22 +46,19 @@ async def test_withdraw_rejects_insufficient_balance() -> None:
|
||||
|
||||
request = Request(scope={"type": "http", "method": "POST"})
|
||||
|
||||
with patch("routstr.core.admin.get_wallet") as mock_wallet, \
|
||||
patch("routstr.core.admin.get_proofs_per_mint_and_unit") as mock_proofs, \
|
||||
patch("routstr.core.admin.slow_filter_spend_proofs") as mock_filter:
|
||||
|
||||
mock_w = Mock()
|
||||
mock_w.keysets = {}
|
||||
mock_w.proofs = []
|
||||
mock_wallet.return_value = mock_w
|
||||
mock_proofs.return_value = []
|
||||
mock_filter.return_value = []
|
||||
|
||||
with patch(
|
||||
"routstr.core.admin.send_token",
|
||||
new=AsyncMock(
|
||||
side_effect=ValueError(
|
||||
"No trusted mint has 1000000 sat available; balances={}"
|
||||
)
|
||||
),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await withdraw(request, WithdrawRequest(amount=1000000, unit="sat"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Insufficient" in str(exc_info.value.detail)
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Insufficient" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from routstr.core import db
|
||||
from routstr.core.db import create_db_engine
|
||||
from routstr.core.settings import settings
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_uses_validated_bounded_pool_settings(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "database_pool_size", 12)
|
||||
monkeypatch.setattr(settings, "database_max_overflow", 3)
|
||||
monkeypatch.setattr(settings, "database_pool_timeout", 2.5)
|
||||
monkeypatch.setattr(settings, "database_pool_recycle", 900)
|
||||
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
|
||||
|
||||
engine = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/pool.db")
|
||||
try:
|
||||
assert engine.pool.size() == 12 # type: ignore[attr-defined]
|
||||
assert engine.pool._max_overflow == 3 # type: ignore[attr-defined]
|
||||
assert engine.pool._timeout == 2.5 # type: ignore[attr-defined]
|
||||
assert engine.pool._recycle == 900
|
||||
assert engine.pool._pre_ping is False
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_sqlite_keeps_static_pool(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "database_pool_pre_ping", True)
|
||||
engine = create_db_engine("sqlite+aiosqlite://")
|
||||
try:
|
||||
assert isinstance(engine.pool, StaticPool)
|
||||
assert engine.pool._pre_ping is True
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def test_non_sqlite_backend_enables_pre_ping_automatically(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
|
||||
fake_engine = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(db, "create_async_engine", return_value=fake_engine) as factory,
|
||||
patch.object(db.event, "listen") as listen,
|
||||
):
|
||||
created = create_db_engine("postgresql+asyncpg://user:pass@db/node")
|
||||
|
||||
assert created is fake_engine
|
||||
assert factory.call_args.kwargs["pool_pre_ping"] is True
|
||||
assert listen.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_created_engine_warns_for_long_checkouts(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "database_pool_hold_warn_seconds", 0.0)
|
||||
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
|
||||
first = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/first.db")
|
||||
second = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/second.db")
|
||||
|
||||
try:
|
||||
with patch.object(db.logger, "warning") as warning:
|
||||
async with first.connect() as connection:
|
||||
await connection.exec_driver_sql("SELECT 1")
|
||||
async with second.connect() as connection:
|
||||
await connection.exec_driver_sql("SELECT 1")
|
||||
|
||||
assert warning.call_count == 2
|
||||
assert all(
|
||||
call.kwargs["extra"]["threshold_seconds"] == 0.0
|
||||
for call in warning.call_args_list
|
||||
)
|
||||
finally:
|
||||
await first.dispose()
|
||||
await second.dispose()
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import AsyncGenerator
|
||||
from typing import Any, AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
@@ -8,7 +9,8 @@ from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.auth import get_reservation_snapshot, pay_for_request
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.upstream.ehbp import (
|
||||
finalize_ehbp_actual_cost_payment,
|
||||
finalize_ehbp_max_cost_payment,
|
||||
@@ -43,18 +45,38 @@ async def _api_key(session: AsyncSession, hashed_key: str) -> ApiKey | None:
|
||||
).one_or_none()
|
||||
|
||||
|
||||
def _fail_nth_api_key_update(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
target_update: int,
|
||||
) -> None:
|
||||
"""Return rowcount=0 for one API-key UPDATE without mutating the database."""
|
||||
original_exec = session.exec
|
||||
api_key_updates = 0
|
||||
|
||||
async def exec_with_failure(
|
||||
statement: Any, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
nonlocal api_key_updates
|
||||
table = getattr(statement, "table", None)
|
||||
if getattr(table, "name", None) == "api_keys":
|
||||
api_key_updates += 1
|
||||
if api_key_updates == target_update:
|
||||
return MagicMock(rowcount=0)
|
||||
return await original_exec(statement, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(session, "exec", exec_with_failure)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="ehbp-actual",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
key = ApiKey(hashed_key="ehbp-actual", balance=10_000)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(key, session)
|
||||
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key,
|
||||
@@ -68,6 +90,7 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
|
||||
"input_msats": 500,
|
||||
"output_msats": 700,
|
||||
},
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated = await _api_key(session, "ehbp-actual")
|
||||
@@ -82,28 +105,22 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
|
||||
async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(
|
||||
hashed_key="ehbp-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
parent = ApiKey(hashed_key="ehbp-parent", balance=10_000)
|
||||
child = ApiKey(
|
||||
hashed_key="ehbp-child",
|
||||
balance=0,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
parent_key_hash="ehbp-parent",
|
||||
hashed_key="ehbp-child", balance=0, parent_key_hash="ehbp-parent"
|
||||
)
|
||||
session.add(parent)
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
await pay_for_request(child, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(child, session)
|
||||
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
child,
|
||||
session,
|
||||
max_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated_parent = await _api_key(session, "ehbp-parent")
|
||||
@@ -123,17 +140,16 @@ async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matches_no_rows(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="ehbp-missing-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
key = ApiKey(hashed_key="ehbp-missing-parent", balance=10_000)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await session.delete(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(key, session)
|
||||
_fail_nth_api_key_update(session, monkeypatch, target_update=1)
|
||||
rollback_spy = AsyncMock(wraps=session.rollback)
|
||||
monkeypatch.setattr(session, "rollback", rollback_spy)
|
||||
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key,
|
||||
@@ -141,45 +157,52 @@ async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matche
|
||||
reserved_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
cost_info={"total_msats": 1_200},
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert await _api_key(session, "ehbp-missing-parent") is None
|
||||
rollback_spy.assert_awaited_once()
|
||||
updated = await _api_key(session, "ehbp-missing-parent")
|
||||
assert updated is not None
|
||||
assert updated.balance == 10_000
|
||||
assert updated.reserved_balance == 3_000
|
||||
assert updated.total_spent == 0
|
||||
release = await session.get(ReservationRelease, reservation.release_id)
|
||||
assert release is not None
|
||||
assert release.status == "active"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_max_cost_payment_rolls_back_parent_when_child_update_matches_no_rows(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
parent = ApiKey(
|
||||
hashed_key="ehbp-rollback-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
parent = ApiKey(hashed_key="ehbp-rollback-parent", balance=10_000)
|
||||
child = ApiKey(
|
||||
hashed_key="ehbp-missing-child",
|
||||
balance=0,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
parent_key_hash="ehbp-rollback-parent",
|
||||
)
|
||||
session.add(parent)
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
await session.delete(child)
|
||||
await session.commit()
|
||||
await pay_for_request(child, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(child, session)
|
||||
_fail_nth_api_key_update(session, monkeypatch, target_update=2)
|
||||
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
child,
|
||||
session,
|
||||
max_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated_parent = await _api_key(session, "ehbp-rollback-parent")
|
||||
assert updated_parent is not None
|
||||
assert updated_parent.balance == 10_000
|
||||
assert updated_parent.reserved_balance == 3_000
|
||||
assert updated_parent.reserved_at == 123
|
||||
assert updated_parent.total_spent == 0
|
||||
assert await _api_key(session, "ehbp-missing-child") is None
|
||||
updated_child = await _api_key(session, "ehbp-missing-child")
|
||||
assert updated_child is not None
|
||||
assert updated_child.reserved_balance == 3_000
|
||||
assert updated_child.total_spent == 0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
@@ -13,9 +13,19 @@ from routstr import wallet
|
||||
from routstr.core import db
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _session_context(session: Mock) -> AsyncIterator[Mock]:
|
||||
yield session
|
||||
class _SessionContext:
|
||||
def __init__(self, session: Mock) -> None:
|
||||
self.session = session
|
||||
|
||||
async def __aenter__(self) -> Mock:
|
||||
return self.session
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _session_context(session: Mock) -> _SessionContext:
|
||||
return _SessionContext(session)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -47,7 +57,7 @@ async def test_fee_payout_checkpoint_is_atomic_and_durable() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_checkpoints_before_sending() -> None:
|
||||
async def test_fee_payout_prepares_wallet_then_checkpoints_before_sending() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
@@ -57,6 +67,10 @@ async def test_fee_payout_checkpoints_before_sending() -> None:
|
||||
payout_wallet = Mock()
|
||||
events: list[str] = []
|
||||
|
||||
async def prepare(*_args: object, **_kwargs: object) -> Mock:
|
||||
events.append("prepare")
|
||||
return payout_wallet
|
||||
|
||||
async def checkpoint(*_args: object) -> bool:
|
||||
events.append("checkpoint")
|
||||
return True
|
||||
@@ -77,18 +91,92 @@ async def test_fee_payout_checkpoints_before_sending() -> None:
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", side_effect=checkpoint),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", side_effect=complete),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=payout_wallet)),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=prepare)),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", side_effect=send),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
assert events == ["checkpoint", "send", "complete"]
|
||||
assert events == ["prepare", "checkpoint", "send", "complete"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_preparation_failure_does_not_checkpoint() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
checkpoint = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", checkpoint),
|
||||
patch(
|
||||
"routstr.wallet.get_wallet",
|
||||
AsyncMock(side_effect=RuntimeError("wallet unavailable")),
|
||||
),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
checkpoint.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_lost_checkpoint_race_does_not_send() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
send = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch(
|
||||
"routstr.wallet.db.reset_routstr_fee",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", send),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
send.assert_not_awaited()
|
||||
warning.assert_called_once_with("Routstr fee payout was already claimed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -107,7 +195,9 @@ async def test_fee_payout_does_not_retry_an_unresolved_checkpoint() -> None:
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock()) as checkpoint,
|
||||
patch("routstr.wallet.get_wallet", AsyncMock()) as get_wallet,
|
||||
@@ -141,7 +231,9 @@ async def test_fee_payout_keeps_checkpoint_when_send_outcome_is_unknown() -> Non
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", complete),
|
||||
@@ -158,3 +250,139 @@ async def test_fee_payout_keeps_checkpoint_when_send_outcome_is_unknown() -> Non
|
||||
|
||||
complete.assert_not_awaited()
|
||||
critical.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_cancellation_during_send_alerts_and_propagates() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
complete = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch("routstr.wallet.asyncio.sleep", AsyncMock(return_value=None)),
|
||||
patch(
|
||||
"routstr.wallet.db.create_session", return_value=_session_context(session)
|
||||
),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", complete),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch(
|
||||
"routstr.wallet.raw_send_to_lnurl",
|
||||
AsyncMock(side_effect=asyncio.CancelledError()),
|
||||
),
|
||||
patch("routstr.wallet.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
complete.assert_not_awaited()
|
||||
critical.assert_called_once()
|
||||
assert critical.call_args.args[0] == (
|
||||
"Routstr fee payout outcome is unknown; manual reconciliation required"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure_site", ["session", "completion"])
|
||||
async def test_fee_payout_completion_failures_use_sent_checkpoint_alert(
|
||||
failure_site: str,
|
||||
) -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
completion = AsyncMock()
|
||||
if failure_site == "session":
|
||||
create_session = Mock(
|
||||
side_effect=[
|
||||
_session_context(session),
|
||||
_session_context(session),
|
||||
RuntimeError("pool unavailable"),
|
||||
]
|
||||
)
|
||||
else:
|
||||
create_session = Mock(return_value=_session_context(session))
|
||||
completion.side_effect = RuntimeError("checkpoint unavailable")
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", create_session),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", completion),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", AsyncMock(return_value=5)),
|
||||
patch("routstr.wallet.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
critical.assert_called_once()
|
||||
assert critical.call_args.args[0] == (
|
||||
"Routstr fee payout sent but checkpoint was not completed"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_releases_db_connection_during_send(tmp_path: object) -> None:
|
||||
"""With pool_size=1, the payout must not hold a connection while the
|
||||
external LNURL send is in flight, or the completion step would starve."""
|
||||
engine = create_async_engine(
|
||||
f"sqlite+aiosqlite:///{tmp_path}/payout.db", pool_size=1, max_overflow=0
|
||||
)
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
async with AsyncSession(engine) as session:
|
||||
session.add(db.RoutstrFee(id=1, accumulated_msats=5_000_000))
|
||||
await session.commit()
|
||||
|
||||
@asynccontextmanager
|
||||
async def create_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
|
||||
async def send(*_args: object, **_kwargs: object) -> int:
|
||||
assert engine.pool.checkedout() == 0 # type: ignore[attr-defined]
|
||||
return 5
|
||||
|
||||
try:
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", create_session),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", side_effect=send),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
fee = await db.get_routstr_fee(session)
|
||||
assert fee.payout_in_progress_msats == 0
|
||||
assert fee.total_paid_msats == 5_000_000
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
@@ -4,6 +4,9 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
|
||||
def _run_alembic(root: Path, database_url: str, revision: str) -> None:
|
||||
env = os.environ.copy()
|
||||
@@ -18,6 +21,40 @@ def _run_alembic(root: Path, database_url: str, revision: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "fresh-node.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
|
||||
_run_alembic(root, database_url, "head")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
version = connection.execute(
|
||||
"SELECT version_num FROM alembic_version"
|
||||
).fetchone()
|
||||
columns = {
|
||||
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
|
||||
}
|
||||
fee = connection.execute(
|
||||
"SELECT id, accumulated_msats, total_paid_msats, last_paid_at, "
|
||||
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
|
||||
).fetchone()
|
||||
|
||||
migration_config = Config(str(root / "alembic.ini"))
|
||||
assert version == (
|
||||
ScriptDirectory.from_config(migration_config).get_current_head(),
|
||||
)
|
||||
assert {
|
||||
"id",
|
||||
"accumulated_msats",
|
||||
"total_paid_msats",
|
||||
"last_paid_at",
|
||||
"payout_in_progress_msats",
|
||||
"payout_started_at",
|
||||
} <= columns
|
||||
assert fee == (1, 0, 0, None, 0, None)
|
||||
|
||||
|
||||
def test_fee_payout_checkpoint_migration_preserves_existing_row(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -44,3 +81,36 @@ def test_fee_payout_checkpoint_migration_preserves_existing_row(
|
||||
).fetchone()
|
||||
|
||||
assert row == (5000, 1000, 123, 0, None)
|
||||
|
||||
|
||||
def test_fee_payout_checkpoint_repair_restores_columns_missing_at_old_head(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
old_head = "7f2843d3f4e4"
|
||||
_run_alembic(root, database_url, old_head)
|
||||
|
||||
# Reproduce a database that was stamped to head after a duplicate-column or
|
||||
# unknown-revision recovery skipped part of the migration chain.
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
connection.execute("ALTER TABLE routstr_fees DROP COLUMN payout_started_at")
|
||||
connection.execute(
|
||||
"ALTER TABLE routstr_fees DROP COLUMN payout_in_progress_msats"
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
_run_alembic(root, database_url, "head")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
columns = {
|
||||
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
|
||||
}
|
||||
row = connection.execute(
|
||||
"SELECT payout_in_progress_msats, payout_started_at "
|
||||
"FROM routstr_fees WHERE id = 1"
|
||||
).fetchone()
|
||||
|
||||
assert {"payout_in_progress_msats", "payout_started_at"} <= columns
|
||||
assert row == (0, None)
|
||||
|
||||
@@ -1,11 +1,33 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator, Generator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.wallet import fetch_all_balances
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_balance_fetch_state() -> Generator[None, None, None]:
|
||||
from routstr import wallet
|
||||
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
yield
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _fake_session(): # type: ignore[no-untyped-def]
|
||||
yield MagicMock()
|
||||
@@ -23,11 +45,13 @@ def _patches( # type: ignore[no-untyped-def]
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit",
|
||||
AsyncMock(return_value=user_balance_msats),
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
AsyncMock(
|
||||
return_value={("http://primary:3338", "sat"): user_balance_msats}
|
||||
),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
]
|
||||
@@ -38,8 +62,9 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
|
||||
"""With empty cashu_mints, balances are still fetched for primary_mint."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "cashu_mints", []), patch.object(
|
||||
settings, "primary_mint", "http://primary:3338"
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
):
|
||||
for p in _patches(proof_amount=1000):
|
||||
p.start()
|
||||
@@ -54,13 +79,314 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
|
||||
assert total_wallet == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(return_value=["sat"]),
|
||||
) as supported_units,
|
||||
):
|
||||
for p in _patches(proof_amount=1000):
|
||||
p.start()
|
||||
try:
|
||||
details, *_ = await fetch_all_balances()
|
||||
finally:
|
||||
patch.stopall()
|
||||
|
||||
supported_units.assert_awaited_once_with("http://mint:3338")
|
||||
assert [detail["unit"] for detail in details] == ["sat"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock()
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances()
|
||||
|
||||
assert details[0]["unit"] == settings.primary_mint_unit
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[0]["retry_after_seconds"] > 0
|
||||
get_wallet.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supported_mint_units_come_from_active_keysets() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _get_supported_mint_units
|
||||
|
||||
# Cashu versions/mints may deserialize keyset units as either strings or
|
||||
# Unit enum-like objects. Both representations must be accepted.
|
||||
sat = MagicMock(active=True, unit="sat")
|
||||
msat = MagicMock(active=False, unit="msat")
|
||||
usd = MagicMock(active=True)
|
||||
usd.unit.name = "usd"
|
||||
wallet = MagicMock()
|
||||
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint_unit", "sat"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
|
||||
):
|
||||
units = await _get_supported_mint_units("http://mint:3338")
|
||||
cached_units = await _get_supported_mint_units("http://mint:3338")
|
||||
|
||||
assert units == ["sat", "usd"]
|
||||
assert cached_units == units
|
||||
wallet._get_keysets.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
first = await fetch_all_balances(units=["sat"])
|
||||
second = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert first[0][0]["error"] == "mint unavailable"
|
||||
assert first[0][0]["error_code"] == "unreachable"
|
||||
assert first[0][0]["retry_after_seconds"] == 60
|
||||
assert second[0][0]["error"] == "mint unavailable"
|
||||
assert second[0][0]["error_code"] == "unreachable"
|
||||
assert get_wallet.await_count == 1
|
||||
warning.assert_called_once()
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=71),
|
||||
patch("routstr.wallet.logger.warning"),
|
||||
):
|
||||
await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert get_wallet.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
|
||||
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
|
||||
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert details[0]["error_code"] == "rate_limited"
|
||||
assert details[0]["retry_after_seconds"] == 60
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _mint_cooldown_remaining
|
||||
|
||||
mint = "http://mint:3338"
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", [mint]),
|
||||
patch.object(settings, "primary_mint", mint),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat", "msat"])
|
||||
cooldown = _mint_cooldown_remaining(mint)
|
||||
|
||||
assert get_wallet.await_count == 1
|
||||
assert warning.call_count == 1
|
||||
assert cooldown == 60
|
||||
assert details[0]["error"] == "mint unavailable"
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[1]["error"] == "Mint is unreachable"
|
||||
assert details[1]["error_code"] == "unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
|
||||
"""Slow mint checks must never run while the balance DB session is open."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
session_open = False
|
||||
mint_calls = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def tracked_session(): # type: ignore[no-untyped-def]
|
||||
nonlocal session_open
|
||||
session_open = True
|
||||
try:
|
||||
yield MagicMock()
|
||||
finally:
|
||||
session_open = False
|
||||
|
||||
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
|
||||
nonlocal mint_calls
|
||||
assert session_open is False
|
||||
mint_calls += 1
|
||||
await asyncio.sleep(0)
|
||||
return proofs
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://one:3338", "http://two:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://one:3338"),
|
||||
patch("routstr.wallet.db.create_session", tracked_session),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
AsyncMock(return_value={}),
|
||||
create=True,
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=1)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=slow_filter),
|
||||
),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat", "msat"])
|
||||
|
||||
assert mint_calls == 4
|
||||
assert all("error" not in detail for detail in details)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_bounds_parallel_mint_checks() -> None:
|
||||
"""A slow mint fleet cannot create an unbounded external-I/O fan-out."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
active = 0
|
||||
peak = 0
|
||||
|
||||
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
|
||||
nonlocal active, peak
|
||||
active += 1
|
||||
peak = max(peak, active)
|
||||
await asyncio.sleep(0.01)
|
||||
active -= 1
|
||||
return proofs
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
settings,
|
||||
"cashu_mints",
|
||||
[f"http://mint-{index}:3338" for index in range(8)],
|
||||
),
|
||||
patch.object(settings, "primary_mint", ""),
|
||||
patch.object(settings, "mint_operation_concurrency", 2),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
AsyncMock(return_value={}),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=slow_filter),
|
||||
),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert len(details) == 8
|
||||
assert peak == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_slow_mints_do_not_exhaust_a_single_connection_pool(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Concurrent slow balance refreshes release the sole DB connection promptly."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
engine = create_async_engine(
|
||||
f"sqlite+aiosqlite:///{tmp_path / 'pool-pressure.db'}",
|
||||
pool_size=1,
|
||||
max_overflow=0,
|
||||
pool_timeout=0.2,
|
||||
)
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
|
||||
@asynccontextmanager
|
||||
async def single_pool_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
|
||||
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
|
||||
await asyncio.sleep(0.3)
|
||||
return proofs
|
||||
|
||||
try:
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://slow:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://slow:3338"),
|
||||
patch.object(settings, "mint_operation_concurrency", 1),
|
||||
patch("routstr.wallet.db.create_session", single_pool_session),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=slow_filter),
|
||||
),
|
||||
):
|
||||
results = await asyncio.gather(
|
||||
*(fetch_all_balances(units=["sat"]) for _ in range(6))
|
||||
)
|
||||
|
||||
assert all("error" not in result[0][0] for result in results)
|
||||
assert engine.pool.checkedout() == 0 # type: ignore[attr-defined]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_reports_liability_when_wallet_is_empty() -> None:
|
||||
"""An empty wallet must not hide outstanding user liabilities."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "cashu_mints", []), patch.object(
|
||||
settings, "primary_mint", "http://primary:3338"
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
):
|
||||
for p in _patches(proof_amount=0, user_balance_msats=5000):
|
||||
p.start()
|
||||
@@ -84,9 +410,10 @@ async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
|
||||
"""primary_mint already in cashu_mints is not inspected twice."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(
|
||||
settings, "cashu_mints", ["http://primary:3338"]
|
||||
), patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://primary:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
):
|
||||
for p in _patches(proof_amount=1000):
|
||||
p.start()
|
||||
try:
|
||||
@@ -98,3 +425,58 @@ async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
|
||||
|
||||
assert [d["mint_url"] for d in details] == ["http://primary:3338"]
|
||||
assert total_wallet == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_degrades_when_liability_read_fails() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
AsyncMock(side_effect=RuntimeError("db pool exhausted")),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=1000)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
),
|
||||
):
|
||||
details, total_wallet, total_user, owner = await fetch_all_balances(
|
||||
units=["sat"]
|
||||
)
|
||||
|
||||
assert details[0]["error"] == "db pool exhausted"
|
||||
assert details[0]["wallet_balance"] == 1000
|
||||
assert details[0]["user_balance"] == 0
|
||||
assert details[0]["owner_balance"] == 0
|
||||
assert (total_wallet, total_user, owner) == (1000, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_liability_error_keeps_more_specific_mint_error() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
AsyncMock(side_effect=RuntimeError("db pool exhausted")),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.get_wallet",
|
||||
AsyncMock(side_effect=RuntimeError("mint down")),
|
||||
),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert details[0]["error"] == "mint down"
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import MintQuoteState, Proof
|
||||
|
||||
from routstr.lightning import (
|
||||
InvoiceRecoverRequest,
|
||||
_invoice_settlement_locks,
|
||||
_is_outputs_already_signed,
|
||||
_mint_invoice_quote,
|
||||
check_invoice_payment,
|
||||
get_invoice_status,
|
||||
recover_invoice,
|
||||
)
|
||||
from routstr.wallet import Wallet
|
||||
|
||||
|
||||
def _invoice(**overrides: object) -> SimpleNamespace:
|
||||
values = {
|
||||
"id": "invoice-1",
|
||||
"payment_hash": "quote-1",
|
||||
"amount_sats": 100,
|
||||
"purpose": "create",
|
||||
"status": "pending",
|
||||
"paid_at": None,
|
||||
"api_key_hash": None,
|
||||
"mint_url": "http://mint:3338",
|
||||
"balance_limit": None,
|
||||
"balance_limit_reset": None,
|
||||
"validity_date": None,
|
||||
"created_at": 1,
|
||||
"expires_at": 2,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
|
||||
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
|
||||
|
||||
|
||||
def _recovery_wallet(
|
||||
error: Exception,
|
||||
*,
|
||||
proofs_before: list[Proof] | None = None,
|
||||
proofs_after: list[Proof] | None = None,
|
||||
) -> Mock:
|
||||
async def load_proofs(*, reload: bool) -> None:
|
||||
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
|
||||
wallet.proofs = list(proofs_after)
|
||||
|
||||
wallet = Mock(
|
||||
mint=AsyncMock(side_effect=error),
|
||||
keysets={"keyset-1": Mock()},
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
load_proofs=AsyncMock(side_effect=load_proofs),
|
||||
proofs=list(proofs_before or []),
|
||||
)
|
||||
return wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
|
||||
proofs_after=[_proof(100, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
|
||||
"keyset-1", to=1, batch=25
|
||||
)
|
||||
assert wallet.load_proofs.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("must not mint"),
|
||||
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
|
||||
invoice = _invoice()
|
||||
error = Exception("backend request 11003 failed")
|
||||
wallet = _recovery_wallet(error)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
assert caught.value is error
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
|
||||
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
|
||||
response = httpx.Response(
|
||||
400,
|
||||
request=request,
|
||||
json={"detail": "outputs have already been signed before", "code": 11003},
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
Wallet.raise_on_error_request(response)
|
||||
|
||||
assert _is_outputs_already_signed(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recovered", [0, 99])
|
||||
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
|
||||
recovered: int,
|
||||
) -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="expected at least 100"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(10_000, "different-quote")],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quote_not_found_is_definitively_unpaid() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(
|
||||
side_effect=Exception("Mint Error: quote not found (Code: 0)")
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"Mint Error: quote not found (Code: 10000)",
|
||||
"Mint Error: quote not found (Code: 01)",
|
||||
"Mint Error: quote not found (Code: 0x10)",
|
||||
],
|
||||
)
|
||||
async def test_quote_not_found_without_exact_code_0_is_not_definitively_unpaid(
|
||||
message: str,
|
||||
) -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(side_effect=Exception(message)))
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quote_not_found_case_insensitive() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(
|
||||
side_effect=Exception("MINT ERROR: Quote Not Found (code 0)")
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_pending_invoice_is_not_minted() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="expired")
|
||||
session = AsyncMock()
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
get_wallet.assert_not_awaited()
|
||||
session.commit.assert_awaited_once()
|
||||
assert _invoice_settlement_locks == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambiguous_invoice_mint_timeout_remains_recoverable() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
state_session = AsyncMock()
|
||||
state_session.exec.return_value.rowcount = 1
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
yield state_session
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert invoice.status == "settlement_pending"
|
||||
state_session.commit.assert_awaited_once()
|
||||
session.rollback.assert_not_awaited()
|
||||
# One commit closes the initial read transaction before external I/O.
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quote_not_found_after_payment_confirmation_is_not_unpaid() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
state_session = AsyncMock()
|
||||
state_session.exec.return_value.rowcount = 1
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
yield state_session
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(
|
||||
side_effect=Exception("Mint Error: quote not found (Code: 0)")
|
||||
),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is False
|
||||
assert invoice.status == "settlement_pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quote_lookup_timeout_is_not_definitively_unpaid() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(side_effect=httpx.TimeoutException("quote timeout"))
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overdue_invoice_does_not_expire_after_ambiguous_quote_lookup() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock(return_value=False)
|
||||
|
||||
with patch("routstr.lightning.check_invoice_payment", check):
|
||||
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
|
||||
|
||||
assert response.status == "pending"
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overdue_invoice_expires_only_after_definitive_unpaid_quote() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock(return_value=True)
|
||||
|
||||
async def expire(
|
||||
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
|
||||
) -> bool:
|
||||
assert definitive is True
|
||||
candidate.status = "expired"
|
||||
return True
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.check_invoice_payment", check),
|
||||
patch(
|
||||
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
|
||||
side_effect=expire,
|
||||
) as expire_invoice,
|
||||
):
|
||||
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
|
||||
|
||||
assert response.status == "expired"
|
||||
expire_invoice.assert_awaited_once_with(invoice, session, True)
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_applies_authoritative_expiry_helper() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
result = Mock()
|
||||
result.first.return_value = invoice
|
||||
session.exec.return_value = result
|
||||
check = AsyncMock(return_value=True)
|
||||
|
||||
async def expire(
|
||||
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
|
||||
) -> bool:
|
||||
assert definitive is True
|
||||
candidate.status = "expired"
|
||||
return True
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.check_invoice_payment", check),
|
||||
patch(
|
||||
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
|
||||
side_effect=expire,
|
||||
) as expire_invoice,
|
||||
):
|
||||
response = await recover_invoice(
|
||||
InvoiceRecoverRequest(bolt11="lnbc-test"), session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert response.status == "expired"
|
||||
expire_invoice.assert_awaited_once_with(invoice, session, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_state_write_failure_still_reports_non_expirable_outcome() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(
|
||||
return_value=Mock(paid=True, state=MintQuoteState.paid)
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
|
||||
),
|
||||
patch(
|
||||
"routstr.lightning.create_session",
|
||||
side_effect=RuntimeError("database unavailable"),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
definitively_unpaid = await check_invoice_payment(
|
||||
invoice, session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert definitively_unpaid is False
|
||||
assert invoice.status == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settlement_pending_invoice_does_not_expire() -> None:
|
||||
invoice = _invoice(status="settlement_pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock()
|
||||
|
||||
with patch("routstr.lightning.check_invoice_payment", check):
|
||||
response = await get_invoice_status(
|
||||
invoice.id, session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
check.assert_awaited_once_with(invoice, session)
|
||||
assert response.status == "settlement_pending"
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
|
||||
async def refresh(obj: SimpleNamespace) -> None:
|
||||
return None
|
||||
|
||||
session.refresh = AsyncMock(side_effect=refresh)
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
owned = AsyncMock()
|
||||
owned.exec.return_value.rowcount = 1
|
||||
yield owned
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(return_value=(True, "b" * 64)),
|
||||
) as finalize,
|
||||
):
|
||||
await asyncio.gather(
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert invoice.status == "paid"
|
||||
finalize.assert_awaited_once()
|
||||
assert _invoice_settlement_locks == {}
|
||||
@@ -1,70 +1,202 @@
|
||||
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
|
||||
|
||||
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
|
||||
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
|
||||
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
|
||||
timeout as LNURLError instead of hanging.
|
||||
"""
|
||||
"""LNURL melt attempts must not misclassify ambiguous payment outcomes."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState
|
||||
|
||||
from routstr.payment import lnurl
|
||||
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
|
||||
from routstr.core.settings import settings
|
||||
from routstr.mint import MintCooldownError, MintRateGuard
|
||||
from routstr.payment.lnurl import (
|
||||
MeltOutcomeAmbiguousError,
|
||||
raw_send_to_lnurl,
|
||||
)
|
||||
|
||||
LNURL_DATA = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
|
||||
def _wallet() -> tuple[MagicMock, list[MagicMock]]:
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
wallet = MagicMock(url="https://mint.test")
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
return wallet, proofs
|
||||
|
||||
|
||||
def _lnurl_patches() -> tuple[Any, Any]:
|
||||
return (
|
||||
patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data",
|
||||
AsyncMock(return_value=LNURL_DATA),
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
async def test_raw_send_to_lnurl_timeout_keeps_unpaid_outcome_ambiguous() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
|
||||
async def _hang(**kwargs: object) -> None:
|
||||
await asyncio.sleep(5) # far longer than the patched timeout
|
||||
await asyncio.sleep(5)
|
||||
|
||||
wallet.melt = AsyncMock(side_effect=_hang)
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.unpaid)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
|
||||
):
|
||||
with pytest.raises(LNURLError, match="Melt timed out"):
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
wallet.set_reserved_for_melt.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
|
||||
"""A prompt melt still returns the net amount, unaffected by the guard."""
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
async def test_raw_send_to_lnurl_timeout_reconciled_paid_is_success() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
wallet.melt = AsyncMock(return_value=MagicMock())
|
||||
async def _hang(**kwargs: object) -> None:
|
||||
await asyncio.sleep(5)
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
wallet.melt = AsyncMock(side_effect=_hang)
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.paid)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
):
|
||||
paid = await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
assert paid > 0
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_pending_response_stays_ambiguous() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.pending))
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.pending)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 5),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
|
||||
):
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("rate_error", ["cooldown", "http_429"])
|
||||
async def test_raw_send_to_lnurl_rate_rejection_unreserves_proofs(
|
||||
rate_error: str,
|
||||
) -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock()
|
||||
wallet.set_reserved_for_send = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
async def run_operation(factory: Any, *, op_name: str, **_: object) -> Any:
|
||||
if op_name == "lnurl_melt":
|
||||
if rate_error == "cooldown":
|
||||
raise MintCooldownError(str(wallet.url), 60)
|
||||
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
|
||||
response = httpx.Response(429, request=request)
|
||||
raise httpx.HTTPStatusError(
|
||||
"rate limited", request=request, response=response
|
||||
)
|
||||
return await factory()
|
||||
|
||||
with (
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
patch(
|
||||
"routstr.payment.lnurl.run_mint_operation",
|
||||
side_effect=run_operation,
|
||||
),
|
||||
pytest.raises((MintCooldownError, httpx.HTTPStatusError)),
|
||||
):
|
||||
await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
wallet.melt.assert_not_awaited()
|
||||
wallet.set_reserved_for_send.assert_awaited_once_with(
|
||||
proofs, reserved=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_mint_wrapper_http_429_unreserves_proofs() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
|
||||
response = httpx.Response(429, request=request)
|
||||
wallet.melt = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"rate limited", request=request, response=response
|
||||
)
|
||||
)
|
||||
wallet.set_reserved_for_send = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(httpx.HTTPStatusError),
|
||||
):
|
||||
await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
wallet.melt.assert_awaited_once()
|
||||
wallet.set_reserved_for_send.assert_awaited_once_with(
|
||||
proofs, reserved=False
|
||||
)
|
||||
MintRateGuard._guards.pop(str(wallet.url), None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_succeeds_on_explicit_paid_response() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.paid))
|
||||
wallet.get_melt_quote = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 5),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
):
|
||||
paid = await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
@@ -72,3 +204,4 @@ async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
|
||||
|
||||
assert paid > 0
|
||||
wallet.melt.assert_awaited_once()
|
||||
wallet.get_melt_quote.assert_not_awaited()
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState, ProofSpentState
|
||||
|
||||
from routstr.wallet import (
|
||||
TokenConsumedError,
|
||||
_confirm_melt_paid,
|
||||
_reconcile_ambiguous_melt,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_quote_is_authoritative_when_proof_lookup_would_fail() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
|
||||
check_proof_state=AsyncMock(side_effect=RuntimeError("proof API unavailable")),
|
||||
)
|
||||
|
||||
assert await _reconcile_ambiguous_melt(wallet, "quote-1", [Mock()]) is True
|
||||
wallet.check_proof_state.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_snapshot_unpaid_unspent_remains_non_retryable() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.unpaid)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=ProofSpentState.unspent)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="ambiguous"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-2", [Mock()])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_pending_melt_response_requires_reconciliation() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.pending)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=ProofSpentState.pending)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="ambiguous"):
|
||||
await _confirm_melt_paid(
|
||||
wallet,
|
||||
"quote-pending",
|
||||
[Mock()],
|
||||
Mock(state=MeltQuoteState.pending),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("quote_state", "proof_state"),
|
||||
[
|
||||
(MeltQuoteState.pending, ProofSpentState.pending),
|
||||
(MeltQuoteState.unpaid, ProofSpentState.spent),
|
||||
(MeltQuoteState.unpaid, ProofSpentState.pending),
|
||||
],
|
||||
)
|
||||
async def test_ambiguous_or_consumed_melt_is_never_reported_unspent(
|
||||
quote_state: MeltQuoteState, proof_state: ProofSpentState
|
||||
) -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=quote_state)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=proof_state)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="reconciliation required"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-3", [Mock()])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_melt_reconciliation_is_non_retryable() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(side_effect=RuntimeError("mint unavailable")),
|
||||
check_proof_state=AsyncMock(),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="outcome is unknown"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-4", [Mock()])
|
||||
@@ -18,6 +18,7 @@ from fastapi.responses import Response, StreamingResponse
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
|
||||
from routstr.auth import ReservationSnapshot # noqa: E402
|
||||
from routstr.core.db import ApiKey # noqa: E402
|
||||
from routstr.payment.cost_calculation import CostData # noqa: E402
|
||||
from routstr.payment.models import Architecture, Model, Pricing # noqa: E402
|
||||
@@ -450,7 +451,8 @@ async def test_non_streaming_dispatches_via_litellm_and_returns_anthropic_respon
|
||||
assert payload["model"] == "openai/gpt-4o-mini" # mapped back to requested
|
||||
assert payload["usage"]["input_tokens"] == 5
|
||||
assert payload["usage"]["output_tokens"] == 3
|
||||
assert payload["usage"]["cost"] == 0.0001
|
||||
assert payload["usage"]["cost"]["total_msats"] == 1234
|
||||
assert payload["usage"]["cost"]["total_usd"] == 0.0001
|
||||
assert payload["usage"]["cost_sats"] == 1
|
||||
|
||||
|
||||
@@ -498,6 +500,12 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
yield {"type": "message_stop"}
|
||||
|
||||
fake_cost = {"total_msats": 4321, "total_usd": 0.00015}
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="messages-stream",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=10_000,
|
||||
)
|
||||
|
||||
captured_cost_call: dict[str, Any] = {}
|
||||
|
||||
@@ -508,9 +516,11 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
max_cost: int,
|
||||
model_obj: Any = None,
|
||||
provider_fee: Any = None,
|
||||
reservation_snapshot: Any = None,
|
||||
) -> dict:
|
||||
captured_cost_call["combined_data"] = combined_data
|
||||
captured_cost_call["max_cost"] = max_cost
|
||||
captured_cost_call["reservation_snapshot"] = reservation_snapshot
|
||||
return fake_cost
|
||||
|
||||
fake_session = MagicMock()
|
||||
@@ -544,6 +554,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
session=session,
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
@@ -566,6 +577,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
assert combined["usage"]["input_tokens"] == 5
|
||||
assert combined["usage"]["output_tokens"] == 7
|
||||
assert combined["model"] == "openai/gpt-4o-mini"
|
||||
assert captured_cost_call["reservation_snapshot"] is reservation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -593,6 +605,12 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||
|
||||
fake_cost = {"total_msats": 999, "total_usd": 0.0001}
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="messages-byte-stream",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=10_000,
|
||||
)
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
async def fake_adjust(
|
||||
@@ -602,8 +620,10 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
max_cost: int,
|
||||
model_obj: Any = None,
|
||||
provider_fee: Any = None,
|
||||
reservation_snapshot: Any = None,
|
||||
) -> dict:
|
||||
captured["combined_data"] = combined_data
|
||||
captured["reservation_snapshot"] = reservation_snapshot
|
||||
return fake_cost
|
||||
|
||||
fake_session = MagicMock()
|
||||
@@ -636,6 +656,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
session=session,
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
@@ -659,6 +680,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
assert combined["usage"]["input_tokens"] == 3
|
||||
assert combined["usage"]["output_tokens"] == 4
|
||||
assert combined["model"] == "openai/gpt-4o-mini"
|
||||
assert captured["reservation_snapshot"] is reservation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -833,6 +855,9 @@ async def test_x_cashu_streaming_replays_events_and_sets_refund_header() -> None
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
assert result.headers.get("X-Cashu") == "cashuSTREAM"
|
||||
assert result.headers.get("X-Routstr-Cost-Msats") == "1500000"
|
||||
assert result.headers.get("X-Routstr-Input-Cost-Msats") == "1000000"
|
||||
assert result.headers.get("X-Routstr-Output-Cost-Msats") == "500000"
|
||||
# 1_500_000 msats → 1500 sats. Refund = 5000 - 1500 = 3500.
|
||||
mock_refund.assert_awaited_once()
|
||||
refund_call = mock_refund.await_args
|
||||
@@ -851,6 +876,9 @@ async def test_x_cashu_streaming_replays_events_and_sets_refund_header() -> None
|
||||
assert "event: message_start" in joined
|
||||
assert "event: message_delta" in joined
|
||||
assert "event: message_stop" in joined
|
||||
assert '"total_msats": 1500000' in joined
|
||||
assert '"input_msats": 1000000' in joined
|
||||
assert '"output_msats": 500000' in joined
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import Unit
|
||||
|
||||
from routstr.mint import (
|
||||
MintCooldownError,
|
||||
MintRateGuard,
|
||||
MintRateLimitedError,
|
||||
fail_fast_mint_operations,
|
||||
)
|
||||
from routstr.wallet import Wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cooldown_fails_fast_while_wallet_mutation_scope_is_held() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(3600, reason="rate_limited")
|
||||
operation = AsyncMock(return_value="should not run")
|
||||
|
||||
with (
|
||||
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
|
||||
pytest.raises(MintCooldownError) as caught,
|
||||
):
|
||||
async with fail_fast_mint_operations():
|
||||
await guard.run(operation)
|
||||
|
||||
assert caught.value.retry_after_seconds > 0
|
||||
operation.assert_not_awaited()
|
||||
sleep.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_cooldown_allows_probe_in_wallet_mutation_scope() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(0, reason="rate_limited")
|
||||
operation = AsyncMock(return_value="recovered")
|
||||
|
||||
async with fail_fast_mint_operations():
|
||||
result = await guard.run(operation)
|
||||
|
||||
assert result == "recovered"
|
||||
operation.assert_awaited_once()
|
||||
assert guard._needs_probe is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_fast_does_not_wait_behind_existing_probe() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(0, reason="rate_limited")
|
||||
probe_started = asyncio.Event()
|
||||
release_probe = asyncio.Event()
|
||||
|
||||
async def probe() -> str:
|
||||
probe_started.set()
|
||||
await release_probe.wait()
|
||||
return "recovered"
|
||||
|
||||
first = asyncio.create_task(guard.run(probe))
|
||||
await probe_started.wait()
|
||||
try:
|
||||
async with fail_fast_mint_operations():
|
||||
with pytest.raises(MintCooldownError):
|
||||
await asyncio.wait_for(guard.run(AsyncMock()), timeout=0.05)
|
||||
finally:
|
||||
release_probe.set()
|
||||
assert await first == "recovered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cashu_429_dispatches_through_wallet_override() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
429,
|
||||
request=request,
|
||||
json={"detail": "too many requests", "code": 42900},
|
||||
)
|
||||
|
||||
wallet = object.__new__(Wallet)
|
||||
wallet.url = "http://mint:3338"
|
||||
wallet.db = Mock()
|
||||
wallet.keysets = {"loaded": Mock()}
|
||||
wallet.mint_info = Mock()
|
||||
wallet.mint_info.requires_blind_auth_path.return_value = False
|
||||
wallet.mint_info.requires_clear_auth_path.return_value = False
|
||||
wallet.auth_db = None
|
||||
wallet.auth_keyset_id = None
|
||||
|
||||
real_client = httpx.AsyncClient
|
||||
|
||||
def client_factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
|
||||
return real_client(
|
||||
transport=httpx.MockTransport(handler),
|
||||
base_url=str(kwargs["base_url"]),
|
||||
)
|
||||
|
||||
with (
|
||||
patch("cashu.wallet.v1_api.httpx.AsyncClient", side_effect=client_factory),
|
||||
pytest.raises(MintRateLimitedError),
|
||||
):
|
||||
await wallet.mint_quote(1, Unit.sat)
|
||||
|
||||
|
||||
async def test_guard_concurrency_change_preserves_cooldown_state() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
mint_url = "https://mint.test-concurrency-carryover"
|
||||
with patch.object(settings, "mint_max_concurrency", 2):
|
||||
guard = MintRateGuard.get(mint_url)
|
||||
guard.apply_cooldown(120.0, reason="rate_limited")
|
||||
guard._consecutive_rate_limits = 3
|
||||
|
||||
with patch.object(settings, "mint_max_concurrency", 5):
|
||||
rebuilt = MintRateGuard.get(mint_url)
|
||||
|
||||
assert rebuilt is not guard
|
||||
assert rebuilt.cooldown_remaining() > 0
|
||||
assert rebuilt._cooldown_reason == "rate_limited"
|
||||
assert rebuilt._consecutive_rate_limits == 3
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Persisted mint preferences must not bypass the configured trusted set."""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core.settings import settings
|
||||
from routstr.lightning import _request_mint_with_fallback
|
||||
|
||||
TRUSTED = "https://good-mint.example.com"
|
||||
UNTRUSTED = "https://removed-mint.example.com"
|
||||
|
||||
|
||||
async def test_untrusted_allowed_mints_fall_back_to_trusted_set() -> None:
|
||||
attempted: list[str] = []
|
||||
|
||||
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
|
||||
attempted.append(mint_url)
|
||||
raise ConnectionError("unreachable in test")
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", TRUSTED),
|
||||
patch.object(settings, "cashu_mints", [TRUSTED]),
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
|
||||
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
|
||||
):
|
||||
with pytest.raises(Exception):
|
||||
await _request_mint_with_fallback(10, allowed_mints=[UNTRUSTED])
|
||||
|
||||
assert UNTRUSTED not in attempted
|
||||
assert attempted == [TRUSTED]
|
||||
|
||||
|
||||
async def test_trusted_allowed_mints_are_used_verbatim() -> None:
|
||||
attempted: list[str] = []
|
||||
|
||||
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
|
||||
attempted.append(mint_url)
|
||||
raise ConnectionError("unreachable in test")
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", TRUSTED),
|
||||
patch.object(settings, "cashu_mints", [TRUSTED, "https://other.example.com"]),
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
|
||||
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
|
||||
):
|
||||
with pytest.raises(Exception):
|
||||
await _request_mint_with_fallback(10, allowed_mints=[TRUSTED])
|
||||
|
||||
assert attempted == [TRUSTED]
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
|
||||
env = os.environ.copy()
|
||||
env["DATABASE_URL"] = database_url
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", command, revision],
|
||||
cwd=root,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice_columns(database_path: Path) -> set[str]:
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
return {
|
||||
row[1]
|
||||
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
|
||||
}
|
||||
|
||||
|
||||
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "mint-url-migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
previous_head = "64ed5594df1f"
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "ecfa0d6e2a36")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "downgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "head")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -125,3 +125,33 @@ async def test_get_max_cost_for_model_tolerance() -> None:
|
||||
"gpt-4", session=mock_session, model_obj=mock_model
|
||||
)
|
||||
assert cost == 450000 # 500 sats * 1000 * 0.9 = 450000
|
||||
|
||||
|
||||
async def test_discounted_max_cost_floors_at_min_request_msat() -> None:
|
||||
from routstr.payment.helpers import calculate_discounted_max_cost
|
||||
|
||||
pricing = Mock()
|
||||
pricing.prompt = 0.001
|
||||
pricing.completion = 0.001
|
||||
pricing.max_prompt_cost = 100.0
|
||||
pricing.max_completion_cost = 100.0
|
||||
|
||||
model_obj = Mock()
|
||||
model_obj.sats_pricing = pricing
|
||||
model_obj.top_provider = None
|
||||
model_obj.context_length = None
|
||||
|
||||
body = {
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 1,
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(settings, "fixed_pricing", False),
|
||||
patch.object(settings, "tolerance_percentage", 0),
|
||||
patch.object(settings, "min_request_msat", 1000),
|
||||
):
|
||||
cost = await calculate_discounted_max_cost(150_000, body, model_obj)
|
||||
|
||||
assert cost == 1000
|
||||
|
||||
@@ -59,23 +59,29 @@ async def test_periodic_payout_includes_primary_mint_not_in_cashu_mints() -> Non
|
||||
get_wallet = AsyncMock(return_value=MagicMock())
|
||||
raw_send = AsyncMock(return_value=1000)
|
||||
|
||||
with patch.object(settings, "cashu_mints", []), patch.object(
|
||||
settings, "primary_mint", "http://primary:3338"
|
||||
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
|
||||
settings, "payout_interval_seconds", _INTERVAL
|
||||
), patch.object(settings, "min_payout_sat", 10), patch(
|
||||
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
|
||||
), patch("routstr.wallet.db.create_session", _fake_session), patch(
|
||||
"routstr.wallet.get_wallet", get_wallet
|
||||
), patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
), patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
), patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
|
||||
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
|
||||
patch.object(settings, "payout_interval_seconds", _INTERVAL),
|
||||
patch.object(settings, "min_payout_sat", 10),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.total_user_liability",
|
||||
AsyncMock(return_value=0),
|
||||
),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
|
||||
):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
@@ -84,12 +90,66 @@ async def test_periodic_payout_includes_primary_mint_not_in_cashu_mints() -> Non
|
||||
assert raw_send.await_count >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_releases_session_before_slow_mint_send() -> None:
|
||||
"""The DB connection is returned before the external LNURL call starts."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
session_open = False
|
||||
sends_completed = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def tracked_session(): # type: ignore[no-untyped-def]
|
||||
nonlocal session_open
|
||||
session_open = True
|
||||
try:
|
||||
yield MagicMock()
|
||||
finally:
|
||||
session_open = False
|
||||
|
||||
async def raw_send(*args: object, **kwargs: object) -> int:
|
||||
nonlocal sends_completed
|
||||
assert session_open is False
|
||||
sends_completed += 1
|
||||
return 1000
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
|
||||
patch.object(settings, "payout_interval_seconds", _INTERVAL),
|
||||
patch.object(settings, "min_payout_sat", 10),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
|
||||
patch("routstr.wallet.db.create_session", tracked_session),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.total_user_liability",
|
||||
AsyncMock(return_value=0),
|
||||
),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", AsyncMock(side_effect=raw_send)),
|
||||
):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
assert sends_completed == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_isolates_failing_mint() -> None:
|
||||
"""A failing mint does not prevent payout for the other mints."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
async def _get_wallet(mint_url: str, unit: str) -> MagicMock:
|
||||
async def _get_wallet(
|
||||
mint_url: str, unit: str, force_reload: bool = False
|
||||
) -> MagicMock:
|
||||
if mint_url == "http://bad:3338":
|
||||
raise RuntimeError("mint unreachable")
|
||||
return MagicMock()
|
||||
@@ -97,23 +157,29 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
|
||||
get_wallet = AsyncMock(side_effect=_get_wallet)
|
||||
raw_send = AsyncMock(return_value=1000)
|
||||
|
||||
with patch.object(
|
||||
settings, "cashu_mints", ["http://bad:3338", "http://good:3338"]
|
||||
), patch.object(settings, "primary_mint", "http://good:3338"), patch.object(
|
||||
settings, "receive_ln_address", "owner@ln.tld"
|
||||
), patch.object(settings, "payout_interval_seconds", _INTERVAL), patch.object(
|
||||
settings, "min_payout_sat", 10
|
||||
), patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()), patch(
|
||||
"routstr.wallet.db.create_session", _fake_session
|
||||
), patch("routstr.wallet.get_wallet", get_wallet), patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
), patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
), patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
|
||||
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://bad:3338", "http://good:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://good:3338"),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
|
||||
patch.object(settings, "payout_interval_seconds", _INTERVAL),
|
||||
patch.object(settings, "min_payout_sat", 10),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.total_user_liability",
|
||||
AsyncMock(return_value=0),
|
||||
),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
|
||||
):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
@@ -128,27 +194,39 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_handles_session_creation_failure() -> None:
|
||||
"""A db.create_session failure is logged and the payout loop continues."""
|
||||
"""A db.create_session failure is logged per mint/unit and the loop continues."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
create_session = MagicMock(side_effect=RuntimeError("db unavailable"))
|
||||
logger = MagicMock()
|
||||
|
||||
with patch.object(settings, "cashu_mints", ["http://mint:3338"]), patch.object(
|
||||
settings, "primary_mint", "http://mint:3338"
|
||||
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
|
||||
settings, "payout_interval_seconds", _INTERVAL
|
||||
), patch(
|
||||
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
|
||||
), patch(
|
||||
"routstr.wallet.db.create_session", create_session
|
||||
), patch("routstr.wallet.logger", logger):
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
|
||||
patch.object(settings, "payout_interval_seconds", _INTERVAL),
|
||||
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
|
||||
patch("routstr.wallet.db.create_session", create_session),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
),
|
||||
patch("routstr.wallet.logger", logger),
|
||||
):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
create_session.assert_called_once()
|
||||
logger.error.assert_called_once()
|
||||
# The liability session is opened per mint/unit (sat + msat), and each
|
||||
# DB failure retains the cycle-specific alert wording while remaining
|
||||
# isolated to its own iteration.
|
||||
assert create_session.call_count == 2
|
||||
assert logger.error.call_count == 2
|
||||
message = logger.error.call_args.args[0]
|
||||
extra = logger.error.call_args.kwargs["extra"]
|
||||
assert message == "Error in periodic payout cycle: RuntimeError"
|
||||
assert extra == {"error": "db unavailable"}
|
||||
assert extra["error"] == "db unavailable"
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from routstr import proxy as proxy_module
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_closes_request_session_before_returning_response() -> None:
|
||||
"""Route completion must release DB resources before response delivery."""
|
||||
request = MagicMock()
|
||||
request.method = "GET"
|
||||
request.headers = {"accept": "application/json"}
|
||||
request.url.path = "/not-an-api-route"
|
||||
request.state.request_id = "test-request"
|
||||
session = AsyncMock()
|
||||
|
||||
response = await proxy_module.proxy(request, "not-an-api-route", session=session)
|
||||
|
||||
assert response.status_code == 404
|
||||
session.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_session_is_closed_before_first_stream_chunk() -> None:
|
||||
request = MagicMock()
|
||||
session = AsyncMock()
|
||||
|
||||
async def stream() -> AsyncIterator[bytes]:
|
||||
session.close.assert_awaited_once()
|
||||
yield b"chunk"
|
||||
|
||||
upstream_response = StreamingResponse(stream())
|
||||
with patch("routstr.proxy._proxy", AsyncMock(return_value=upstream_response)):
|
||||
response = await proxy_module.proxy(
|
||||
request, "v1/chat/completions", session=session
|
||||
)
|
||||
|
||||
assert isinstance(response, StreamingResponse)
|
||||
chunks = [chunk async for chunk in response.body_iterator]
|
||||
assert chunks == [b"chunk"]
|
||||
@@ -1,4 +1,6 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
@@ -7,6 +9,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr import wallet
|
||||
from routstr.core.db import CashuTransaction
|
||||
from routstr.wallet import refund_sweep_once
|
||||
|
||||
@@ -39,6 +42,43 @@ async def _load(
|
||||
return {row.token: row for row in result.all()}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_releases_db_session_during_token_redemption(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="eligible", amount=1, unit="sat", type="out", created_at=800
|
||||
),
|
||||
)
|
||||
session_open = False
|
||||
|
||||
@asynccontextmanager
|
||||
async def tracked_session() -> AsyncIterator[AsyncSession]:
|
||||
nonlocal session_open
|
||||
async with session_factory() as session:
|
||||
session_open = True
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session_open = False
|
||||
|
||||
async def receive_token(token: str) -> None:
|
||||
assert token == "eligible"
|
||||
assert session_open is False
|
||||
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", tracked_session),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=receive_token)),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
assert (await _load(session_factory))["eligible"].swept is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_only_processes_expired_eligible_outgoing_tokens(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
@@ -90,14 +130,17 @@ async def test_refund_sweep_only_processes_expired_eligible_outgoing_tokens(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("error", "collected"),
|
||||
("error", "collected", "claim_started_at"),
|
||||
[
|
||||
(RuntimeError("token already spent"), True),
|
||||
(RuntimeError("mint unavailable"), False),
|
||||
(RuntimeError("token already spent"), True, None),
|
||||
(RuntimeError("mint unavailable"), False, 1000),
|
||||
],
|
||||
)
|
||||
async def test_refund_sweep_records_terminal_but_not_transient_failures(
|
||||
session_factory: async_sessionmaker[AsyncSession], error: Exception, collected: bool
|
||||
async def test_refund_sweep_records_spent_and_unknown_outcomes_safely(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
error: Exception,
|
||||
collected: bool,
|
||||
claim_started_at: int | None,
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
@@ -116,3 +159,236 @@ async def test_refund_sweep_records_terminal_but_not_transient_failures(
|
||||
refund = (await _load(session_factory))["refund"]
|
||||
assert refund.collected is collected
|
||||
assert refund.swept is False
|
||||
assert refund.sweep_started_at == claim_started_at
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_spend_failure_retains_claim_and_stale_retry_records_sweep(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="post-spend-failure",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
AsyncMock(
|
||||
side_effect=wallet.TokenConsumedError(
|
||||
"Mint on primary failed after successful melt"
|
||||
)
|
||||
),
|
||||
),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
retained = (await _load(session_factory))["post-spend-failure"]
|
||||
assert retained.swept is False
|
||||
assert retained.collected is False
|
||||
assert retained.sweep_started_at == 1000
|
||||
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
|
||||
patch("routstr.wallet.time.time", return_value=1300),
|
||||
patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
AsyncMock(side_effect=RuntimeError("token already spent")),
|
||||
),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
recovered = (await _load(session_factory))["post-spend-failure"]
|
||||
assert recovered.swept is True
|
||||
assert recovered.collected is False
|
||||
assert recovered.sweep_started_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_retains_claim_on_cancellation_during_redemption(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="cancelled", amount=1, unit="sat", type="out", created_at=800
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
AsyncMock(side_effect=asyncio.CancelledError()),
|
||||
),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await refund_sweep_once()
|
||||
|
||||
refund = (await _load(session_factory))["cancelled"]
|
||||
assert refund.swept is False
|
||||
assert refund.sweep_started_at == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checkpoint_failure_retains_claim_and_stale_retry_records_sweep(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="checkpoint-failure",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
),
|
||||
)
|
||||
real_set_state = wallet._set_refund_sweep_state
|
||||
|
||||
async def fail_swept_checkpoint(
|
||||
refund_id: str,
|
||||
*,
|
||||
predicates: tuple[object, ...] = (),
|
||||
**values: object,
|
||||
) -> int:
|
||||
if values.get("swept") is True:
|
||||
raise RuntimeError("checkpoint unavailable")
|
||||
return await real_set_state(refund_id, predicates=predicates, **values)
|
||||
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch(
|
||||
"routstr.wallet.recieve_token", AsyncMock(return_value=(1, "sat", "mint"))
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet._set_refund_sweep_state",
|
||||
side_effect=fail_swept_checkpoint,
|
||||
),
|
||||
patch("routstr.wallet.logger.critical") as critical,
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
retained = (await _load(session_factory))["checkpoint-failure"]
|
||||
assert retained.swept is False
|
||||
assert retained.collected is False
|
||||
assert retained.sweep_started_at == 1000
|
||||
critical.assert_called_once()
|
||||
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
|
||||
patch("routstr.wallet.time.time", return_value=1300),
|
||||
patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
AsyncMock(side_effect=RuntimeError("token already spent")),
|
||||
),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
recovered = (await _load(session_factory))["checkpoint-failure"]
|
||||
assert recovered.swept is True
|
||||
assert recovered.collected is False
|
||||
assert recovered.sweep_started_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("redemption_succeeds", [True, False])
|
||||
async def test_expired_worker_cannot_overwrite_or_release_newer_claim(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
redemption_succeeds: bool,
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="reclaimed",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
),
|
||||
)
|
||||
|
||||
async def replace_claim(_token: str) -> tuple[int, str, str]:
|
||||
async with session_factory() as session:
|
||||
result = await session.exec(
|
||||
select(CashuTransaction).where(CashuTransaction.token == "reclaimed")
|
||||
)
|
||||
transaction = result.one()
|
||||
transaction.sweep_started_at = 1100
|
||||
session.add(transaction)
|
||||
await session.commit()
|
||||
if not redemption_succeeds:
|
||||
raise RuntimeError("mint unavailable")
|
||||
return (1, "sat", "mint")
|
||||
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=replace_claim)),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
reclaimed = (await _load(session_factory))["reclaimed"]
|
||||
assert reclaimed.swept is False
|
||||
assert reclaimed.collected is False
|
||||
assert reclaimed.sweep_started_at == 1100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_recovers_stale_claim_without_misreporting_collection(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="stale",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
sweep_started_at=100,
|
||||
),
|
||||
CashuTransaction(
|
||||
token="active",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
sweep_started_at=950,
|
||||
),
|
||||
)
|
||||
receive = AsyncMock(side_effect=RuntimeError("token already spent"))
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch("routstr.wallet.recieve_token", receive),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
receive.assert_awaited_once_with("stale")
|
||||
loaded = await _load(session_factory)
|
||||
assert loaded["stale"].swept is True
|
||||
assert loaded["stale"].collected is False
|
||||
assert loaded["stale"].sweep_started_at is None
|
||||
assert loaded["active"].swept is False
|
||||
assert loaded["active"].sweep_started_at == 950
|
||||
|
||||
@@ -62,6 +62,101 @@ def test_payout_settings_have_sensible_defaults() -> None:
|
||||
assert s.payout_interval_seconds == 900
|
||||
|
||||
|
||||
def test_database_pool_defaults_provide_concurrency_headroom() -> None:
|
||||
s = Settings()
|
||||
assert s.database_pool_size == 10
|
||||
assert s.database_max_overflow == 20
|
||||
assert s.database_pool_timeout == 15.0
|
||||
assert s.database_pool_recycle == 1800
|
||||
assert s.database_pool_pre_ping is False
|
||||
assert s.database_pool_hold_warn_seconds == 10.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "bad_value"),
|
||||
[
|
||||
("database_pool_size", 0),
|
||||
("database_max_overflow", -1),
|
||||
("database_pool_timeout", 0),
|
||||
("database_pool_recycle", -1),
|
||||
("database_pool_hold_warn_seconds", 0),
|
||||
],
|
||||
)
|
||||
def test_database_pool_settings_reject_invalid_values(
|
||||
field: str, bad_value: int
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
Settings.parse_obj({field: bad_value})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_pool_fields_are_env_only_not_persisted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""DB pool sizing is infrastructure the node needs *before* it can read the
|
||||
DB, so it can never be configured from the DB — it must never be written to
|
||||
the settings blob, and a stale/injected DB value must never shadow env.
|
||||
"""
|
||||
monkeypatch.setenv("DATABASE_POOL_SIZE", "7")
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
s = await SettingsService.initialize(session)
|
||||
|
||||
# The env value is live for runtime consumers...
|
||||
assert s.database_pool_size == 7
|
||||
# ...but pool sizing is never written to the settings blob.
|
||||
blob = await _read_settings_blob(session)
|
||||
for field in (
|
||||
"database_pool_size",
|
||||
"database_max_overflow",
|
||||
"database_pool_timeout",
|
||||
"database_pool_recycle",
|
||||
"database_pool_pre_ping",
|
||||
"database_pool_hold_warn_seconds",
|
||||
):
|
||||
assert field not in blob
|
||||
|
||||
# Even a stale blob that somehow carries a pool value must not win: env
|
||||
# stays authoritative on the next initialize.
|
||||
await session.exec( # type: ignore
|
||||
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
|
||||
d=json.dumps({"database_pool_size": 99})
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
again = await SettingsService.initialize(session)
|
||||
assert again.database_pool_size == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_does_not_apply_env_only_fields_to_live_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""DB pool sizing is env-only: a settings update must neither persist it nor
|
||||
mutate the live value. The engine pool is already built at boot from env, so
|
||||
a UI/API update carrying a pool value must not make the live setting diverge
|
||||
from the running pool.
|
||||
"""
|
||||
monkeypatch.delenv("DATABASE_POOL_SIZE", raising=False)
|
||||
monkeypatch.setattr(settings, "database_pool_size", 10)
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
await SettingsService.initialize(session)
|
||||
await SettingsService.update(
|
||||
{"database_pool_size": 99, "name": "PoolTweaker"}, session
|
||||
)
|
||||
|
||||
# A non-env-only field still updates normally...
|
||||
assert settings.name == "PoolTweaker"
|
||||
# ...but the env-only pool size stays at the boot value.
|
||||
assert settings.database_pool_size == 10
|
||||
# ...and it is never written to the settings blob.
|
||||
blob = await _read_settings_blob(session)
|
||||
assert "database_pool_size" not in blob
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,bad_value",
|
||||
[
|
||||
|
||||
@@ -16,13 +16,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.auth import pay_for_request
|
||||
from routstr.balance import refund_wallet_endpoint
|
||||
from routstr.core.db import (
|
||||
ApiKey,
|
||||
ReservationRelease,
|
||||
release_stale_reservations,
|
||||
reset_all_reserved_balances,
|
||||
)
|
||||
@@ -70,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(hashed_key="parentkey", balance=10_000)
|
||||
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
|
||||
session.add(parent)
|
||||
@@ -150,6 +153,39 @@ async def test_release_stale_reservations_releases_old(session: AsyncSession) ->
|
||||
assert key.reserved_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_targeted_parent_cleanup_releases_child_owned_reservation(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(hashed_key="stale-parent", balance=5_000)
|
||||
child = ApiKey(
|
||||
hashed_key="stale-child", parent_key_hash=parent.hashed_key, balance=0
|
||||
)
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 1_000, session)
|
||||
reservation = (
|
||||
await session.exec(
|
||||
select(ReservationRelease).where(
|
||||
ReservationRelease.key_hash == child.hashed_key
|
||||
)
|
||||
)
|
||||
).one()
|
||||
reservation.created_at = int(time.time()) - 1_000
|
||||
session.add(reservation)
|
||||
await session.commit()
|
||||
|
||||
released = await release_stale_reservations(
|
||||
session, max_age_seconds=300, key_hash=parent.hashed_key
|
||||
)
|
||||
|
||||
assert released == 1
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
assert parent.reserved_balance == 0
|
||||
assert child.reserved_balance == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) -> None:
|
||||
key = ApiKey(
|
||||
@@ -170,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
# Reservations without a timestamp may belong to instances running older
|
||||
# code (rolling deploy) — the background sweeper must not touch them.
|
||||
key = ApiKey(
|
||||
@@ -190,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="resetkey",
|
||||
balance=5_000,
|
||||
@@ -352,6 +392,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
upstream.forward_request = AsyncMock(side_effect=asyncio.CancelledError())
|
||||
|
||||
session = MagicMock()
|
||||
reservation_snapshot = MagicMock()
|
||||
revert_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
@@ -369,13 +410,16 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation_snapshot),
|
||||
),
|
||||
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await proxy_module.proxy(request, "v1/chat/completions", session=session)
|
||||
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000)
|
||||
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
@@ -67,6 +68,12 @@ async def test_stream_with_id_injection() -> None:
|
||||
max_cost_for_model=100,
|
||||
background_tasks=background_tasks,
|
||||
requested_model="test-model",
|
||||
reservation_snapshot=ReservationSnapshot(
|
||||
release_id="test-release",
|
||||
key_hash="test_hash",
|
||||
billing_key_hash="test_hash",
|
||||
reserved_msats=100,
|
||||
),
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
import routstr.auth as auth_module
|
||||
from routstr.auth import (
|
||||
ReservationSnapshot,
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
release_reservation,
|
||||
)
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.payment.cost_calculation import MaxCostData
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
|
||||
async def _engine() -> AsyncEngine:
|
||||
engine = create_async_engine("sqlite+aiosqlite://")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
return engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_reservation_is_durable_and_idempotent() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert record is not None and record.status == "active"
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
|
||||
await session.refresh(key)
|
||||
await session.refresh(record)
|
||||
assert key.reserved_balance == 0
|
||||
assert key.reserved_at is None
|
||||
assert record.status == "released"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_only_owns_its_concurrent_reservation() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
|
||||
await pay_for_request(key, 400, session)
|
||||
first = await get_reservation_snapshot(key, session)
|
||||
await pay_for_request(key, 400, session)
|
||||
second = await get_reservation_snapshot(key, session)
|
||||
|
||||
assert first.release_id != second.release_id
|
||||
assert await release_reservation(first, session, 400) is True
|
||||
assert await release_reservation(first, session, 400) is True
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == 400
|
||||
|
||||
assert await release_reservation(second, session, 400) is True
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == 0
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_updates_parent_and_child_atomically() -> None:
|
||||
engine = await _engine()
|
||||
parent = ApiKey(hashed_key="parent", balance=1_000)
|
||||
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 500, session)
|
||||
snapshot = await get_reservation_snapshot(child, session)
|
||||
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
assert (parent.reserved_balance, child.reserved_balance) == (0, 0)
|
||||
assert (parent.reserved_at, child.reserved_at) == (None, None)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_rolls_back_partial_parent_child_update() -> None:
|
||||
engine = await _engine()
|
||||
parent = ApiKey(hashed_key="parent", balance=1_000)
|
||||
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 500, session)
|
||||
snapshot = await get_reservation_snapshot(child, session)
|
||||
child.reserved_balance = 100
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
|
||||
assert await release_reservation(snapshot, session, 500) is False
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert (parent.reserved_balance, child.reserved_balance) == (500, 100)
|
||||
assert record is not None and record.status == "active"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_commit_failure_cannot_release_charged_reservation() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
cost = MaxCostData(
|
||||
base_msats=500,
|
||||
input_msats=0,
|
||||
output_msats=0,
|
||||
total_msats=500,
|
||||
)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
with (
|
||||
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
|
||||
patch.object(
|
||||
session,
|
||||
"refresh",
|
||||
AsyncMock(side_effect=SQLAlchemyError("post-commit refresh failed")),
|
||||
),
|
||||
):
|
||||
with pytest.raises(SQLAlchemyError, match="post-commit refresh failed"):
|
||||
await adjust_payment_for_tokens(key, {}, session, 500)
|
||||
|
||||
await session.rollback()
|
||||
assert await release_reservation(snapshot, session, 500) is False
|
||||
charged_key = await session.get(ApiKey, "key")
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert charged_key is not None
|
||||
assert (charged_key.balance, charged_key.reserved_balance) == (500, 0)
|
||||
assert record is not None and record.status == "charged"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_background_settlement_uses_explicit_reservation() -> None:
|
||||
engine = await _engine()
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key", provider_fee=1.0
|
||||
)
|
||||
key = ApiKey(hashed_key="generic-key", balance=1_000)
|
||||
cost = MaxCostData(
|
||||
base_msats=500,
|
||||
input_msats=0,
|
||||
output_msats=0,
|
||||
total_msats=500,
|
||||
)
|
||||
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
context_token = auth_module._current_reservation.set(None)
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.base.create_session",
|
||||
side_effect=lambda: AsyncSession(engine, expire_on_commit=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
auth_module.adjust_payment_for_tokens,
|
||||
),
|
||||
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
|
||||
):
|
||||
await provider._finalize_generic_streaming_payment(
|
||||
key.hashed_key,
|
||||
500,
|
||||
"audio/speech",
|
||||
model_obj=None,
|
||||
provider_fee=provider.provider_fee,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
finally:
|
||||
auth_module._current_reservation.reset(context_token)
|
||||
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
settled_key = await session.get(ApiKey, key.hashed_key)
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert settled_key is not None
|
||||
assert (settled_key.balance, settled_key.reserved_balance) == (500, 0)
|
||||
assert record is not None and record.status == "charged"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_release_is_terminal_and_suppresses_background_charge() -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
|
||||
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
upstream_response = MagicMock()
|
||||
upstream_response.status_code = 200
|
||||
upstream_response.headers = {"content-type": "text/event-stream"}
|
||||
upstream_response.aiter_bytes = aiter_bytes
|
||||
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "test-key-hash"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
release = AsyncMock(return_value=True)
|
||||
reservation_snapshot = MagicMock()
|
||||
reservation_snapshot.reserved_msats = 500
|
||||
background_tasks = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
AsyncMock(side_effect=SQLAlchemyError("database unavailable")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.base.get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation_snapshot),
|
||||
),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
response = await provider.handle_streaming_chat_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
background_tasks=background_tasks,
|
||||
)
|
||||
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for _ in response.body_iterator:
|
||||
pass
|
||||
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(reservation_snapshot, session, 500)
|
||||
background_tasks.add_task.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"release_outcome",
|
||||
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
|
||||
)
|
||||
async def test_responses_streaming_releases_and_raises_on_billing_failure(
|
||||
release_outcome: bool | BaseException,
|
||||
) -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
|
||||
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
|
||||
yield (
|
||||
b'data: {"type":"response.completed","response":{"model":"test",'
|
||||
b'"usage":{"input_tokens":1,"output_tokens":1}}}\n\n'
|
||||
)
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
upstream_response = MagicMock(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
upstream_response.aiter_bytes = aiter_bytes
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "responses-key"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
snapshot = ReservationSnapshot(
|
||||
release_id="responses-release",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=500,
|
||||
)
|
||||
release = (
|
||||
AsyncMock(side_effect=release_outcome)
|
||||
if isinstance(release_outcome, BaseException)
|
||||
else AsyncMock(return_value=release_outcome)
|
||||
)
|
||||
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
|
||||
|
||||
with (
|
||||
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
response = await provider.handle_streaming_responses_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
emitted = bytearray()
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for chunk in response.body_iterator:
|
||||
if isinstance(chunk, str):
|
||||
emitted.extend(chunk.encode())
|
||||
else:
|
||||
emitted.extend(bytes(chunk))
|
||||
|
||||
assert b'"total_msats": 0' not in emitted
|
||||
adjust.assert_awaited_once()
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(snapshot, session, 500)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("via_litellm", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"release_outcome",
|
||||
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
|
||||
)
|
||||
async def test_messages_streaming_releases_and_raises_on_billing_failure(
|
||||
via_litellm: bool,
|
||||
release_outcome: bool | BaseException,
|
||||
) -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "messages-key"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
snapshot = ReservationSnapshot(
|
||||
release_id=f"messages-{'litellm' if via_litellm else 'native'}",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=500,
|
||||
)
|
||||
release = (
|
||||
AsyncMock(side_effect=release_outcome)
|
||||
if isinstance(release_outcome, BaseException)
|
||||
else AsyncMock(return_value=release_outcome)
|
||||
)
|
||||
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
|
||||
|
||||
async def native_chunks() -> AsyncGenerator[bytes, None]:
|
||||
yield (
|
||||
b'event: message_start\ndata: {"type":"message_start","message":'
|
||||
b'{"model":"test","usage":{"input_tokens":1,"output_tokens":0}}}\n\n'
|
||||
)
|
||||
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||
|
||||
async def litellm_chunks() -> AsyncGenerator[dict, None]:
|
||||
yield {
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"model": "test",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 0},
|
||||
},
|
||||
}
|
||||
yield {"type": "message_stop"}
|
||||
|
||||
with (
|
||||
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
if via_litellm:
|
||||
response = provider._stream_litellm_messages(
|
||||
iterator=litellm_chunks(),
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
requested_model=None,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
else:
|
||||
upstream_response = MagicMock(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
upstream_response.aiter_bytes = native_chunks
|
||||
response = await provider.handle_streaming_messages_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for _ in response.body_iterator:
|
||||
pass
|
||||
|
||||
adjust.assert_awaited_once()
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(snapshot, session, 500)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_key_reservation_snapshot_is_rejected_without_mutation() -> None:
|
||||
engine = await _engine()
|
||||
first = ApiKey(hashed_key="first", balance=1_000)
|
||||
second = ApiKey(hashed_key="second", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(first)
|
||||
session.add(second)
|
||||
await session.commit()
|
||||
await pay_for_request(first, 500, session)
|
||||
snapshot = await get_reservation_snapshot(first, session)
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not belong"):
|
||||
await adjust_payment_for_tokens(
|
||||
second,
|
||||
{"model": "test", "usage": None},
|
||||
session,
|
||||
500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
await session.refresh(first)
|
||||
await session.refresh(second)
|
||||
assert first.reserved_balance == 500
|
||||
assert second.reserved_balance == 0
|
||||
|
||||
await engine.dispose()
|
||||
@@ -24,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.upstream import base
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
@@ -67,6 +68,12 @@ async def _drive(chunks: list[bytes], requested_model: str | None = None) -> lis
|
||||
max_cost_for_model=100,
|
||||
background_tasks=MagicMock(),
|
||||
requested_model=requested_model,
|
||||
reservation_snapshot=ReservationSnapshot(
|
||||
release_id="test-release",
|
||||
key_hash="test_hash",
|
||||
billing_key_hash="test_hash",
|
||||
reserved_msats=100,
|
||||
),
|
||||
)
|
||||
|
||||
out: list[bytes] = []
|
||||
|
||||
@@ -331,6 +331,7 @@ async def test_5xx_wrapped_rate_limit_is_classified(
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
from routstr import proxy as proxy_module
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.core.exceptions import UpstreamError
|
||||
|
||||
@@ -359,6 +360,12 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="rate-limit-release",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=1_000,
|
||||
)
|
||||
revert_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
@@ -376,10 +383,13 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation),
|
||||
),
|
||||
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
|
||||
):
|
||||
response = await proxy_module.proxy(
|
||||
@@ -396,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
# Single upstream failed -> reservation reverted exactly once (no double-charge).
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000)
|
||||
revert_mock.assert_awaited_once_with(key, session, 1000, reservation)
|
||||
|
||||
+1586
-48
File diff suppressed because it is too large
Load Diff
@@ -67,8 +67,13 @@ async def test_non_streaming_includes_cost_sats() -> None:
|
||||
)
|
||||
|
||||
body = json.loads(response.body)
|
||||
assert "cost_sats" in body["usage"]
|
||||
assert body["usage"]["cost_sats"] == 5 # 5000 msats // 1000
|
||||
assert body["usage"]["cost"]["total_msats"] == 5000
|
||||
assert body["usage"]["cost"]["input_msats"] == 3000
|
||||
assert body["usage"]["cost"]["output_msats"] == 2000
|
||||
assert response.headers["x-routstr-cost-msats"] == "5000"
|
||||
assert response.headers["x-routstr-input-cost-msats"] == "3000"
|
||||
assert response.headers["x-routstr-output-cost-msats"] == "2000"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -96,7 +101,7 @@ async def test_non_streaming_cost_sats_value_rounds_down() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_preserves_existing_usage_fields() -> None:
|
||||
async def test_non_streaming_preserves_tokens_and_replaces_upstream_cost() -> None:
|
||||
provider = _make_provider()
|
||||
cost_data = _make_cost_data(total_msats=3000)
|
||||
|
||||
@@ -127,7 +132,8 @@ async def test_non_streaming_preserves_existing_usage_fields() -> None:
|
||||
assert usage["prompt_tokens"] == 100
|
||||
assert usage["completion_tokens"] == 50
|
||||
assert usage["total_tokens"] == 150
|
||||
assert usage["cost"] == 0.00015
|
||||
assert usage["cost"]["total_msats"] == 3000
|
||||
assert usage["cost"]["total_usd"] == 0.00025
|
||||
assert usage["cost_sats"] == 3
|
||||
|
||||
|
||||
|
||||
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
|
||||
const formatMintLabel = (detail: BalanceDetail) =>
|
||||
`${detail.mint_url.replace('https://', '').replace('http://', '')} • ${detail.unit.toUpperCase()}`;
|
||||
|
||||
const formatBalanceError = (detail: BalanceDetail) => {
|
||||
const labels: Record<string, string> = {
|
||||
rate_limited: 'rate limited',
|
||||
unreachable: 'unreachable',
|
||||
cooldown: 'cooling down',
|
||||
mint_error: 'mint error',
|
||||
};
|
||||
const label =
|
||||
(detail.error_code ? labels[detail.error_code] : undefined) ??
|
||||
detail.error ??
|
||||
'error';
|
||||
const retryAfter = detail.retry_after_seconds;
|
||||
return retryAfter && retryAfter > 0
|
||||
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
|
||||
: label;
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Card>
|
||||
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
|
||||
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
|
||||
{formatMintLabel(detail)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
<TableCell
|
||||
className='text-right font-mono'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Wallet
|
||||
</p>
|
||||
<p className='font-mono text-sm'>
|
||||
<p
|
||||
className='font-mono text-sm'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -91,25 +91,27 @@ export function CashuPaymentWorkflow({
|
||||
setIsCreatingKey(true);
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
const requestPayload: {
|
||||
initial_balance_token: string;
|
||||
balance_limit?: number;
|
||||
balance_limit_reset?: string;
|
||||
validity_date?: number;
|
||||
} = {
|
||||
initial_balance_token: initialToken.trim(),
|
||||
});
|
||||
if (balanceLimit) params.append('balance_limit', balanceLimit);
|
||||
};
|
||||
if (balanceLimit) requestPayload.balance_limit = Number(balanceLimit);
|
||||
if (balanceLimitReset)
|
||||
params.append('balance_limit_reset', balanceLimitReset);
|
||||
requestPayload.balance_limit_reset = balanceLimitReset;
|
||||
if (validityDate) {
|
||||
const timestamp = Math.floor(
|
||||
requestPayload.validity_date = Math.floor(
|
||||
new Date(validityDate + 'T23:59:59').getTime() / 1000
|
||||
);
|
||||
params.append('validity_date', timestamp.toString());
|
||||
}
|
||||
const response = await fetch(
|
||||
`${baseUrl}/v1/balance/create?${params.toString()}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}
|
||||
);
|
||||
const response = await fetch(`${baseUrl}/v1/balance/create`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestPayload),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(errorText || 'Failed to create API key');
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AdminService } from '@/lib/api/services/admin';
|
||||
import type { PPQAutoTopupState } from '@/lib/api/services/admin';
|
||||
import type {
|
||||
AdminModel,
|
||||
ProviderModels,
|
||||
@@ -20,12 +22,16 @@ import {
|
||||
Trash2,
|
||||
Key,
|
||||
RotateCcw,
|
||||
AlertTriangle,
|
||||
Unlock,
|
||||
Loader2,
|
||||
} from 'lucide-react';
|
||||
import { ProviderBalance } from '@/components/provider-balance';
|
||||
import { ProviderModelsPanel } from '@/components/provider-models-panel';
|
||||
import { RoutstrCreateKeySection } from '@/components/providers/RoutstrCreateKeySection';
|
||||
import { RoutstrProviderService } from '@/lib/api/services/routstr-provider';
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { getErrorStatus } from '@/lib/api/client';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { cn } from '@/lib/utils';
|
||||
@@ -36,6 +42,16 @@ import {
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
AlertDialogCancel,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from '@/components/ui/alert-dialog';
|
||||
|
||||
interface ProviderCardProps {
|
||||
provider: UpstreamProvider;
|
||||
@@ -77,8 +93,71 @@ export function ProviderCard({
|
||||
}: ProviderCardProps) {
|
||||
const queryClient = useQueryClient();
|
||||
const [isKeyModalOpen, setIsKeyModalOpen] = useState(false);
|
||||
const [isReleaseDialogOpen, setIsReleaseDialogOpen] = useState(false);
|
||||
// The claim as the query cache held it when the admin opened the dialog.
|
||||
// The mutation sends this token rather than re-reading the query at submit
|
||||
// time: a background refetch after the dialog opened must not swap in a
|
||||
// state the admin never saw. The server rejects a stale token with a 409,
|
||||
// which is the authoritative guard.
|
||||
const [reviewedState, setReviewedState] = useState<PPQAutoTopupState | null>(
|
||||
null
|
||||
);
|
||||
const hasDetails = Boolean(provider.api_version) || isExpanded;
|
||||
const isRoutstr = provider.provider_type === 'routstr';
|
||||
const isPPQ = provider.provider_type === 'ppqai';
|
||||
|
||||
const { data: ppqAutoTopupState, isError: ppqStateFetchFailed } = useQuery({
|
||||
queryKey: ['ppq-auto-topup-state', provider.id],
|
||||
queryFn: () => AdminService.getPPQAutoTopupState(provider.id),
|
||||
enabled: isPPQ,
|
||||
refetchInterval: 30000,
|
||||
});
|
||||
|
||||
// A claim the server will not let us release: a worker is between reserving
|
||||
// proofs and hearing back from the mint, and sweeping it would let the next
|
||||
// cycle pay a second invoice.
|
||||
const isPPQPaymentInFlight =
|
||||
Boolean(ppqAutoTopupState?.active) &&
|
||||
ppqAutoTopupState?.releasable === false;
|
||||
|
||||
const openReleaseDialog = () => {
|
||||
setReviewedState(ppqAutoTopupState ?? null);
|
||||
setIsReleaseDialogOpen(true);
|
||||
};
|
||||
|
||||
const releasePPQMutation = useMutation({
|
||||
mutationFn: () =>
|
||||
AdminService.releasePPQAutoTopup(
|
||||
provider.id,
|
||||
reviewedState?.state_token ?? null
|
||||
),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['ppq-auto-topup-state', provider.id],
|
||||
});
|
||||
setIsReleaseDialogOpen(false);
|
||||
setReviewedState(null);
|
||||
toast.success('PPQ auto top-up claim released');
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['ppq-auto-topup-state', provider.id],
|
||||
});
|
||||
if (getErrorStatus(error) === 409) {
|
||||
// The claim changed since it was reviewed; the stale snapshot is
|
||||
// useless, so force a fresh review.
|
||||
setIsReleaseDialogOpen(false);
|
||||
setReviewedState(null);
|
||||
toast.error(
|
||||
'PPQ claim changed since it was reviewed; reopen to see the new state'
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Transient failure: keep the dialog and the reviewed snapshot so the
|
||||
// admin can retry without re-navigating.
|
||||
toast.error(`Failed to release PPQ claim: ${error.message}`);
|
||||
},
|
||||
});
|
||||
|
||||
const refundMutation = useMutation({
|
||||
mutationFn: () => RoutstrProviderService.refundBalance(provider.id),
|
||||
@@ -113,6 +192,35 @@ export function ProviderCard({
|
||||
>
|
||||
{provider.enabled ? 'Enabled' : 'Disabled'}
|
||||
</Badge>
|
||||
{ppqAutoTopupState?.active && (
|
||||
<Badge
|
||||
variant='outline'
|
||||
className={cn(
|
||||
'w-fit gap-1',
|
||||
isPPQPaymentInFlight
|
||||
? 'border-blue-500 text-blue-700 dark:text-blue-400'
|
||||
: 'border-amber-500 text-amber-700 dark:text-amber-400'
|
||||
)}
|
||||
>
|
||||
{isPPQPaymentInFlight ? (
|
||||
<Loader2 className='h-3 w-3 animate-spin' />
|
||||
) : (
|
||||
<AlertTriangle className='h-3 w-3' />
|
||||
)}
|
||||
{isPPQPaymentInFlight
|
||||
? 'Paying invoice'
|
||||
: 'Auto top-up needs review'}
|
||||
</Badge>
|
||||
)}
|
||||
{isPPQ && ppqStateFetchFailed && (
|
||||
<Badge
|
||||
variant='outline'
|
||||
className='border-destructive text-destructive w-fit gap-1'
|
||||
>
|
||||
<AlertTriangle className='h-3 w-3' />
|
||||
Top-up status unavailable
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
<CardDescription className='break-all'>
|
||||
{provider.base_url}
|
||||
@@ -153,6 +261,19 @@ export function ProviderCard({
|
||||
</Button>
|
||||
)}
|
||||
|
||||
{isPPQ && ppqAutoTopupState?.active && !isPPQPaymentInFlight && (
|
||||
<Button
|
||||
variant='outline'
|
||||
size='sm'
|
||||
onClick={openReleaseDialog}
|
||||
className='justify-center gap-1.5 border-amber-500 text-amber-700 dark:text-amber-400'
|
||||
title='Release only after manually verifying the Lightning payment outcome'
|
||||
>
|
||||
<Unlock className='h-4 w-4' />
|
||||
<span>Release top-up</span>
|
||||
</Button>
|
||||
)}
|
||||
|
||||
{isRoutstr && provider.api_key && (
|
||||
<Button
|
||||
variant='outline'
|
||||
@@ -212,6 +333,42 @@ export function ProviderCard({
|
||||
</div>
|
||||
</CardHeader>
|
||||
|
||||
<AlertDialog
|
||||
open={isReleaseDialogOpen}
|
||||
onOpenChange={setIsReleaseDialogOpen}
|
||||
>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Release PPQ auto top-up?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
Only continue after checking PPQ and the Cashu mint and confirming
|
||||
the previous Lightning payment cannot later settle. Releasing an
|
||||
ambiguous payment can allow a duplicate top-up.
|
||||
{reviewedState?.invoice_id
|
||||
? ` Invoice: ${reviewedState.invoice_id}`
|
||||
: ''}
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
onClick={(e) => {
|
||||
// Radix closes the dialog on action click by default; the
|
||||
// mutation handlers decide whether it closes (kept open on
|
||||
// transient errors so the admin can retry).
|
||||
e.preventDefault();
|
||||
releasePPQMutation.mutate();
|
||||
}}
|
||||
disabled={releasePPQMutation.isPending}
|
||||
>
|
||||
{releasePPQMutation.isPending
|
||||
? 'Releasing...'
|
||||
: 'I verified it is safe to retry'}
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
<Dialog open={isKeyModalOpen} onOpenChange={setIsKeyModalOpen}>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogHeader>
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { ProviderFormFields } from '@/components/provider-form-fields';
|
||||
import { ppqAutoTopupSettingsInvalid } from '@/components/providers/PPQAutoTopupSettings';
|
||||
|
||||
interface ProviderFormDialogContentProps {
|
||||
mode: 'create' | 'edit';
|
||||
@@ -52,6 +53,11 @@ export function ProviderFormDialogContent({
|
||||
isSubmitting,
|
||||
availableMints,
|
||||
}: ProviderFormDialogContentProps) {
|
||||
// The server re-validates these bounds; this only stops submitting a form
|
||||
// whose inline errors are already visible.
|
||||
const hasInvalidSettings =
|
||||
formData.provider_type === 'ppqai' &&
|
||||
ppqAutoTopupSettingsInvalid(formData.provider_settings || {});
|
||||
return (
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogHeader>
|
||||
@@ -80,7 +86,7 @@ export function ProviderFormDialogContent({
|
||||
</Button>
|
||||
<Button
|
||||
onClick={onSubmit}
|
||||
disabled={isSubmitting}
|
||||
disabled={isSubmitting || hasInvalidSettings}
|
||||
className='w-full sm:w-auto'
|
||||
>
|
||||
{isSubmitting ? submittingLabel : submitLabel}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { PPQAutoTopupSettings } from '@/components/providers/PPQAutoTopupSettings';
|
||||
import { RoutstrNodeSettings } from '@/components/providers/RoutstrNodeSettings';
|
||||
import { RoutstrCreateKeySection } from '@/components/providers/RoutstrCreateKeySection';
|
||||
|
||||
@@ -78,6 +79,8 @@ export function ProviderFormFields({
|
||||
provider_type: value,
|
||||
base_url: getDefaultBaseUrl(value),
|
||||
provider_fee: value === 'openrouter' ? 1.06 : 1.01,
|
||||
provider_settings:
|
||||
value === prev.provider_type ? prev.provider_settings : {},
|
||||
}));
|
||||
}}
|
||||
>
|
||||
@@ -118,6 +121,19 @@ export function ProviderFormFields({
|
||||
/>
|
||||
)}
|
||||
|
||||
{formData.provider_type === 'ppqai' && (
|
||||
<PPQAutoTopupSettings
|
||||
settings={formData.provider_settings || {}}
|
||||
onSettingsChange={(settings) =>
|
||||
setFormData((prev) => ({
|
||||
...prev,
|
||||
provider_settings: settings,
|
||||
}))
|
||||
}
|
||||
idPrefix={mode === 'edit' ? 'edit' : ''}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className='grid gap-2'>
|
||||
<Label htmlFor={`${idPrefix}slug`}>
|
||||
Slug {mode === 'create' ? '(optional, auto-generated)' : ''}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
'use client';
|
||||
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
|
||||
interface ProviderSettings {
|
||||
auto_topup?: boolean;
|
||||
topup_threshold?: number;
|
||||
topup_amount_limit?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface PPQAutoTopupSettingsProps {
|
||||
settings: ProviderSettings;
|
||||
onSettingsChange: (settings: ProviderSettings) => void;
|
||||
idPrefix?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Field-level validation shared with the dialog's submit gating. The server
|
||||
* enforces the same bounds authoritatively; this only keeps a knowingly
|
||||
* invalid form from being submitted.
|
||||
*
|
||||
* Validation only applies while auto top-up is enabled: a disabled toggle
|
||||
* hides the fields, and stale out-of-range values behind it must not block
|
||||
* submission invisibly. When enabled, both fields are required — a blank
|
||||
* field would otherwise submit and fail server-side.
|
||||
*/
|
||||
export function ppqAutoTopupSettingsErrors(settings: ProviderSettings): {
|
||||
thresholdError?: string;
|
||||
amountError?: string;
|
||||
} {
|
||||
if (!settings.auto_topup) {
|
||||
return {};
|
||||
}
|
||||
const threshold = settings.topup_threshold;
|
||||
const amount = settings.topup_amount_limit;
|
||||
return {
|
||||
thresholdError:
|
||||
threshold === undefined
|
||||
? 'Required when auto top-up is enabled'
|
||||
: threshold <= 0
|
||||
? 'Must be greater than 0'
|
||||
: undefined,
|
||||
amountError:
|
||||
amount === undefined
|
||||
? 'Required when auto top-up is enabled'
|
||||
: amount < 1 || amount > 500
|
||||
? 'Must be between 1 and 500 USD'
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function ppqAutoTopupSettingsInvalid(
|
||||
settings: ProviderSettings
|
||||
): boolean {
|
||||
const { thresholdError, amountError } = ppqAutoTopupSettingsErrors(settings);
|
||||
return Boolean(thresholdError || amountError);
|
||||
}
|
||||
|
||||
export function PPQAutoTopupSettings({
|
||||
settings,
|
||||
onSettingsChange,
|
||||
idPrefix = '',
|
||||
}: PPQAutoTopupSettingsProps) {
|
||||
const prefix = idPrefix ? `${idPrefix}_` : '';
|
||||
const update = (patch: Partial<ProviderSettings>) =>
|
||||
onSettingsChange({ ...settings, ...patch });
|
||||
|
||||
/**
|
||||
* Clearing the field yields '' and parse* yields NaN, which JSON.stringify
|
||||
* turns into null. Drop the key instead so the server rejects a missing
|
||||
* value rather than storing a broken one.
|
||||
*/
|
||||
const updateNumber = (
|
||||
key: 'topup_threshold' | 'topup_amount_limit',
|
||||
raw: string,
|
||||
parse: (value: string) => number
|
||||
) => {
|
||||
const next = { ...settings };
|
||||
const parsed = parse(raw);
|
||||
if (raw === '' || Number.isNaN(parsed)) {
|
||||
delete next[key];
|
||||
} else {
|
||||
next[key] = parsed;
|
||||
}
|
||||
onSettingsChange(next);
|
||||
};
|
||||
|
||||
const { thresholdError, amountError } = ppqAutoTopupSettingsErrors(settings);
|
||||
|
||||
return (
|
||||
<div className='bg-muted/30 grid gap-4 rounded-lg border p-4'>
|
||||
<Label className='text-sm font-semibold'>PPQ Auto Top-up</Label>
|
||||
|
||||
<div className='flex items-center justify-between'>
|
||||
<Label htmlFor={`${prefix}ppq_auto_topup`} className='text-sm'>
|
||||
Enable Auto Top-up
|
||||
</Label>
|
||||
<Switch
|
||||
id={`${prefix}ppq_auto_topup`}
|
||||
checked={!!settings.auto_topup}
|
||||
onCheckedChange={(checked) => update({ auto_topup: checked })}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{settings.auto_topup && (
|
||||
<div className='border-primary/20 grid gap-4 border-l-2 pt-2 pl-4'>
|
||||
<div className='grid gap-2'>
|
||||
<Label
|
||||
htmlFor={`${prefix}ppq_topup_threshold`}
|
||||
className='text-xs font-medium'
|
||||
>
|
||||
When credits are below (USD)
|
||||
</Label>
|
||||
<Input
|
||||
id={`${prefix}ppq_topup_threshold`}
|
||||
type='number'
|
||||
min='0.01'
|
||||
step='0.01'
|
||||
className='h-9'
|
||||
placeholder='e.g. 5'
|
||||
value={settings.topup_threshold ?? ''}
|
||||
aria-invalid={Boolean(thresholdError)}
|
||||
aria-describedby={
|
||||
thresholdError
|
||||
? `${prefix}ppq_topup_threshold_error`
|
||||
: undefined
|
||||
}
|
||||
onChange={(e) =>
|
||||
updateNumber('topup_threshold', e.target.value, parseFloat)
|
||||
}
|
||||
/>
|
||||
{thresholdError && (
|
||||
<p
|
||||
id={`${prefix}ppq_topup_threshold_error`}
|
||||
className='text-destructive text-[10px]'
|
||||
>
|
||||
{thresholdError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className='grid gap-2'>
|
||||
<Label
|
||||
htmlFor={`${prefix}ppq_topup_amount_limit`}
|
||||
className='text-xs font-medium'
|
||||
>
|
||||
Purchase this amount (USD)
|
||||
</Label>
|
||||
<Input
|
||||
id={`${prefix}ppq_topup_amount_limit`}
|
||||
type='number'
|
||||
min='1'
|
||||
max='500'
|
||||
step='1'
|
||||
className='h-9'
|
||||
placeholder='e.g. 10'
|
||||
value={settings.topup_amount_limit ?? ''}
|
||||
aria-invalid={Boolean(amountError)}
|
||||
aria-describedby={
|
||||
amountError
|
||||
? `${prefix}ppq_topup_amount_limit_error`
|
||||
: undefined
|
||||
}
|
||||
onChange={(e) =>
|
||||
updateNumber('topup_amount_limit', e.target.value, (v) =>
|
||||
parseInt(v, 10)
|
||||
)
|
||||
}
|
||||
/>
|
||||
{amountError && (
|
||||
<p
|
||||
id={`${prefix}ppq_topup_amount_limit_error`}
|
||||
className='text-destructive text-[10px]'
|
||||
>
|
||||
{amountError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className='text-muted-foreground text-[10px]'>
|
||||
Pays PPQ's Lightning invoice from the sufficiently funded Cashu
|
||||
mint with the highest available balance.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -161,13 +161,11 @@ export function RoutstrCreateKeySection({
|
||||
|
||||
setIsCreatingCashu(true);
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
initial_balance_token: cashuToken.trim(),
|
||||
const resp = await fetch(`${cleanUrl}/v1/balance/create`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ initial_balance_token: cashuToken.trim() }),
|
||||
});
|
||||
const resp = await fetch(
|
||||
`${cleanUrl}/v1/balance/create?${params.toString()}`,
|
||||
{ method: 'GET', headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
|
||||
if (!resp.ok) {
|
||||
const errorText = await resp.text();
|
||||
|
||||
@@ -144,3 +144,18 @@ export class ApiError extends Error {
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP status of a caught request error, whatever shape it arrived in.
|
||||
* apiClient methods rethrow raw Axios errors, so callers must not rely on
|
||||
* `instanceof ApiError` alone to read a status code.
|
||||
*/
|
||||
export function getErrorStatus(error: unknown): number | undefined {
|
||||
if (error instanceof ApiError) {
|
||||
return error.status;
|
||||
}
|
||||
if (axios.isAxiosError(error)) {
|
||||
return error.response?.status;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -1013,6 +1013,32 @@ export class AdminService {
|
||||
}>(`/admin/api/upstream-providers/${providerId}/balance`);
|
||||
}
|
||||
|
||||
static async getPPQAutoTopupState(
|
||||
providerId: number
|
||||
): Promise<PPQAutoTopupState> {
|
||||
return await apiClient.get<PPQAutoTopupState>(
|
||||
`/admin/api/upstream-providers/${providerId}/ppq-auto-topup`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* `stateToken` must be the `state_token` snapshotted when the admin opened
|
||||
* the confirmation — not re-read at submit time. The server rejects a
|
||||
* release whose claim changed in any way since that snapshot.
|
||||
*/
|
||||
static async releasePPQAutoTopup(
|
||||
providerId: number,
|
||||
stateToken: string | null
|
||||
): Promise<{
|
||||
ok: boolean;
|
||||
released: boolean;
|
||||
}> {
|
||||
return await apiClient.post<{ ok: boolean; released: boolean }>(
|
||||
`/admin/api/upstream-providers/${providerId}/ppq-auto-topup/release`,
|
||||
{ confirmed_safe_to_retry: true, state_token: stateToken }
|
||||
);
|
||||
}
|
||||
|
||||
// ── CLI Tokens ──
|
||||
|
||||
static async listCliTokens(): Promise<CliTokenListItem[]> {
|
||||
@@ -1244,6 +1270,30 @@ export interface TransactionsResponse {
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface PPQAutoTopupState {
|
||||
ok: boolean;
|
||||
active: boolean;
|
||||
/**
|
||||
* Opaque version of the claim as reviewed. Echo it back verbatim to
|
||||
* release; the server rejects a release whose claim changed in any way
|
||||
* (new attempt, phase change, renewed lease) since this was read.
|
||||
*/
|
||||
state_token?: string | null;
|
||||
/** Identifies the attempt currently holding the claim. Informational. */
|
||||
operation_id?: string | null;
|
||||
/** 'claimed' | 'in_flight' | 'reconcile'. Null when the claim is malformed. */
|
||||
phase?: string | null;
|
||||
/** False while a payment is in flight — the server rejects a release then. */
|
||||
releasable?: boolean;
|
||||
expires_at?: number | null;
|
||||
invoice_id?: string | null;
|
||||
created_at?: number;
|
||||
amount?: number;
|
||||
unit?: string;
|
||||
mint_url?: string | null;
|
||||
malformed?: boolean;
|
||||
}
|
||||
|
||||
export interface LightningInvoice {
|
||||
id: string;
|
||||
bolt11: string;
|
||||
|
||||
@@ -36,10 +36,13 @@ export interface BalanceDetail {
|
||||
user_balance: number;
|
||||
owner_balance: number;
|
||||
error?: string;
|
||||
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
|
||||
retry_after_seconds?: number;
|
||||
}
|
||||
|
||||
export interface WithdrawResponse {
|
||||
token: string;
|
||||
mint_url: string;
|
||||
}
|
||||
|
||||
export interface CreateChildKeyResponse {
|
||||
|
||||
Reference in New Issue
Block a user